def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new leaf scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
:type enable_health_checker: enables health checking of children.
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating leaf scheduler: %s" % scheduler_id)
self.lock = threading.RLock()
self._latch = CountUpDownLatch()
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT,
MAX_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._hosts = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._initialize_services(scheduler_id)
self._health_checker = None
self._enable_health_checker = enable_health_checker
self._configured = ConfigStates.UNINITIALIZED
class BranchScheduler(BaseScheduler):
"""Branch scheduler manages child schedulers."""
# TODO(vspivak): introduce a dynamic timeout depending on where we
# are in the tree. For now assume only 3 levels, making the branch
# schedulers always at level 2.
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def _initialize_services(self, scheduler_id):
""" initializes all the services required by this
scheduler. This allows any test classes to override
the services.
"""
self._threadpool = common.services.get(ThreadPoolExecutor)
self._scheduler_client = SchedulerClient()
def configure(self, schedulers):
"""Configure the branch scheduler.
:param schedulers: list of child scheduler ids
:type schedulers: list of ChildInfo
"""
# Transfer children's constraints from list to set, so searching
# elements are more efficient.
self._schedulers = []
for scheduler in schedulers:
self._schedulers.append(ChildInfo.from_thrift(scheduler))
self._coalesce_resources(self._schedulers)
def find(self, request):
"""Find the specified resource.
:type request: FindRequest
:rtype: FindResponse
"""
futures = []
for scheduler in self._schedulers:
future = self._threadpool.submit(
self._find_worker, scheduler.address, scheduler.port,
scheduler.id, copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(futures, timeout=FIND_TIMEOUT)
self._logger.info("Find responses received: %d, timed out: %d",
len(done), len(not_done))
for future in done:
response = future.result()
if response.result == FindResultCode.OK:
return response
return FindResponse(FindResultCode.NOT_FOUND)
def place(self, request):
"""Place the specified resources.
:type request: PlaceRequest
:rtype: PlaceResponse
"""
constraints = self._collect_constraints(request.resource)
self._logger.info("Constraints: %s", constraints)
selected = self._placement_schedulers(request, constraints)
if len(selected) == 0:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
done = self._execute_placement(selected, request)
responses = []
had_resource_constraint = False
for future in done:
response = future.result()
if response.result == PlaceResultCode.OK:
responses.append(response)
elif response.result == PlaceResultCode.RESOURCE_CONSTRAINT:
had_resource_constraint = True
best_response = self._scorer.score(responses)
if best_response is not None:
return best_response
elif had_resource_constraint:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
else:
return PlaceResponse(PlaceResultCode.SYSTEM_ERROR)
def _execute_placement(self, schedulers, request):
futures = []
for scheduler in schedulers:
future = self._threadpool.submit(
self._place_worker, scheduler.address, scheduler.port,
scheduler.id, copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(
futures, timeout=INITIAL_PLACE_TIMEOUT)
self._logger.info("Initial place responses received: %d, "
"timed out: %d", len(done), len(not_done))
if len(done) < MIN_PLACE_FAN_OUT:
self._logger.info("Waiting for more place responses")
done, not_done = concurrent.futures.wait(
futures, timeout=PLACE_TIMEOUT - INITIAL_PLACE_TIMEOUT)
self._logger.info("Total place responses received: %d, "
"timed out: %d", len(done), len(not_done))
return done
def _placement_schedulers(self, request, constraints):
return self._place_strategy.filter_child(self._schedulers,
request,
constraints)
def _find_worker(self, address, port, scheduler_id, request):
"""Invokes Scheduler.find on a single scheduler.
:type address: str
:type port: str
:type scheduler_id: str
:type request: FindRequest
:rtype: FindResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(address, port, scheduler_id,
client_timeout=FIND_TIMEOUT) as client:
return client.find(request)
def _place_worker(self, address, port, scheduler_id, request):
"""Invokes Host.find on a single agent.
:type address: str
:type port: int
:type scheduler_id: str
:type request: PlaceRequest
:rtype: PlaceResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(scheduler_id, address, port,
client_timeout=PLACE_TIMEOUT) as client:
return client.place(request)
def cleanup(self):
assert("Not implemented")
def test_ratio(self, scores, ratio, expected):
scorer = DefaultScorer(ut_ratio=ratio)
place_responses = [PlaceResponse(score=score) for score in scores]
response = scorer.score(place_responses)
assert_that(response.score, is_(expected))
def setUp(self):
self.scorer = DefaultScorer(ut_ratio=9)
class ScorerTestCase(unittest.TestCase):
def setUp(self):
self.scorer = DefaultScorer(ut_ratio=9)
def test_highest_score_random(self):
place_responses = []
for i in xrange(10):
resp = PlaceResponse(
score=Score(random.randint(0, 100), random.randint(0, 100)))
place_responses.append(resp)
self._test_highest_score(place_responses)
@parameterized.expand([
([Score(40, 90), Score(49, 78),
Score(50, 70)], 9, Score(50, 70)),
([Score(40, 90), Score(49, 80),
Score(50, 70)], 9, Score(49, 80)),
([Score(40, 90), Score(45, 80),
Score(50, 70)], 1, Score(40, 90)),
([Score(40, 90), Score(45, 86),
Score(50, 70)], 1, Score(45, 86)),
([Score(40, 60), Score(59, 69),
Score(50, 70)], 0.1, Score(50, 70)),
([Score(40, 60), Score(61, 69),
Score(50, 70)], 0.1, Score(61, 69)),
])
def test_ratio(self, scores, ratio, expected):
scorer = DefaultScorer(ut_ratio=ratio)
place_responses = [PlaceResponse(score=score) for score in scores]
response = scorer.score(place_responses)
assert_that(response.score, is_(expected))
def _test_highest_score(self, place_responses):
response = self.scorer.score(place_responses)
scores = [
self.scorer.score_formula(resp.score) for resp in place_responses
]
assert_that(self.scorer.score_formula(response.score),
is_(max(scores)))
def test_new_resource_score(self):
response = self.scorer.score([
PlaceResponse(score=Score(40, 0)),
PlaceResponse(score=Score(50, 0))
])
# should pick one with higher utilization score
assert_that(response, is_(PlaceResponse(score=Score(50, 0))))
def test_same_score_pick_random(self):
responseA = PlaceResponse(score=Score(98, 100))
responseB = PlaceResponse(score=Score(98, 100))
a_freq = 0
b_freq = 0
# Still can fail with 1/2^30 probability, should be negligible
for i in xrange(0, 30):
response = self.scorer.score([responseA, responseB])
if response is responseA:
a_freq += 1
else:
b_freq += 1
assert_that(a_freq, greater_than(0))
assert_that(b_freq, greater_than(0))
@parameterized.expand([
([Score(60, 0), Score(61, 0), Score(62, 0)], 1, [Score(62, 0)]),
([Score(50, 90), Score(50, 90),
Score(49, 90)], 1, [Score(50, 90)]),
([Score(60, 0), Score(70, 0),
Score(69, 0)], 1, [Score(69, 0), Score(70, 0)]),
([Score(50, 90), Score(45, 80),
Score(10, 70)], 9, [Score(50, 90), Score(45, 80)]),
])
def test_random_scorer_close_scores(self, scores, ut_ratio, expected):
scorer = RandomScorer(ut_ratio=ut_ratio)
place_responses = [PlaceResponse(score=score) for score in scores]
response = scorer.score(place_responses)
assert_that(expected, has_item(response.score))
class BranchScheduler(BaseScheduler):
"""Branch scheduler manages child schedulers."""
# TODO(vspivak): introduce a dynamic timeout depending on where we
# are in the tree. For now assume only 3 levels, making the branch
# schedulers always at level 2.
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def _initialize_services(self, scheduler_id):
""" initializes all the services required by this
scheduler. This allows any test classes to override
the services.
"""
self._threadpool = common.services.get(ThreadPoolExecutor)
self._scheduler_client = SchedulerClient()
def configure(self, schedulers):
"""Configure the branch scheduler.
:param schedulers: list of child scheduler ids
:type schedulers: list of ChildInfo
"""
# Transfer children's constraints from list to set, so searching
# elements are more efficient.
self._schedulers = []
for scheduler in schedulers:
self._schedulers.append(ChildInfo.from_thrift(scheduler))
self._coalesce_resources(self._schedulers)
def find(self, request):
"""Find the specified resource.
:type request: FindRequest
:rtype: FindResponse
"""
futures = []
for scheduler in self._schedulers:
future = self._threadpool.submit(self._find_worker,
scheduler.address, scheduler.port,
scheduler.id,
copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(futures, timeout=FIND_TIMEOUT)
self._logger.info("Find responses received: %d, timed out: %d",
len(done), len(not_done))
for future in done:
response = future.result()
if response.result == FindResultCode.OK:
return response
return FindResponse(FindResultCode.NOT_FOUND)
def place(self, request):
"""Place the specified resources.
:type request: PlaceRequest
:rtype: PlaceResponse
"""
constraints = self._collect_constraints(request.resource)
self._logger.info("Constraints: %s", constraints)
selected = self._placement_schedulers(request, constraints)
if len(selected) == 0:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
done = self._execute_placement(selected, request)
responses = []
had_resource_constraint = False
for future in done:
response = future.result()
if response.result == PlaceResultCode.OK:
responses.append(response)
elif response.result == PlaceResultCode.RESOURCE_CONSTRAINT:
had_resource_constraint = True
best_response = self._scorer.score(responses)
if best_response is not None:
return best_response
elif had_resource_constraint:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
else:
return PlaceResponse(PlaceResultCode.SYSTEM_ERROR)
def _execute_placement(self, schedulers, request):
futures = []
for scheduler in schedulers:
future = self._threadpool.submit(self._place_worker,
scheduler.address, scheduler.port,
scheduler.id,
copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(futures,
timeout=INITIAL_PLACE_TIMEOUT)
self._logger.info(
"Initial place responses received: %d, "
"timed out: %d", len(done), len(not_done))
if len(done) < MIN_PLACE_FAN_OUT:
self._logger.info("Waiting for more place responses")
done, not_done = concurrent.futures.wait(futures,
timeout=PLACE_TIMEOUT -
INITIAL_PLACE_TIMEOUT)
self._logger.info(
"Total place responses received: %d, "
"timed out: %d", len(done), len(not_done))
return done
def _placement_schedulers(self, request, constraints):
return self._place_strategy.filter_child(self._schedulers, request,
constraints)
def _find_worker(self, address, port, scheduler_id, request):
"""Invokes Scheduler.find on a single scheduler.
:type address: str
:type port: str
:type scheduler_id: str
:type request: FindRequest
:rtype: FindResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(address,
port,
scheduler_id,
client_timeout=FIND_TIMEOUT) as client:
return client.find(request)
def _place_worker(self, address, port, scheduler_id, request):
"""Invokes Host.find on a single agent.
:type address: str
:type port: int
:type scheduler_id: str
:type request: PlaceRequest
:rtype: PlaceResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(scheduler_id,
address,
port,
client_timeout=PLACE_TIMEOUT) as client:
return client.place(request)
def cleanup(self):
assert ("Not implemented")
def test_ratio(self, scores, ratio, expected):
scorer = DefaultScorer(ut_ratio=ratio)
place_responses = [PlaceResponse(score=score) for score in scores]
response = scorer.score(place_responses)
assert_that(response.score, is_(expected))
def setUp(self):
self.scorer = DefaultScorer(ut_ratio=9)
class ScorerTestCase(unittest.TestCase):
def setUp(self):
self.scorer = DefaultScorer(ut_ratio=9)
def test_highest_score_random(self):
place_responses = []
for i in xrange(10):
resp = PlaceResponse(score=Score(random.randint(0, 100),
random.randint(0, 100)))
place_responses.append(resp)
self._test_highest_score(place_responses)
@parameterized.expand([
([Score(40, 90), Score(49, 78), Score(50, 70)], 9, Score(50, 70)),
([Score(40, 90), Score(49, 80), Score(50, 70)], 9, Score(49, 80)),
([Score(40, 90), Score(45, 80), Score(50, 70)], 1, Score(40, 90)),
([Score(40, 90), Score(45, 86), Score(50, 70)], 1, Score(45, 86)),
([Score(40, 60), Score(59, 69), Score(50, 70)], 0.1, Score(50, 70)),
([Score(40, 60), Score(61, 69), Score(50, 70)], 0.1, Score(61, 69)),
])
def test_ratio(self, scores, ratio, expected):
scorer = DefaultScorer(ut_ratio=ratio)
place_responses = [PlaceResponse(score=score) for score in scores]
response = scorer.score(place_responses)
assert_that(response.score, is_(expected))
def _test_highest_score(self, place_responses):
response = self.scorer.score(place_responses)
scores = [self.scorer.score_formula(resp.score) for resp in
place_responses]
assert_that(self.scorer.score_formula(response.score), is_(max(
scores)))
def test_new_resource_score(self):
response = self.scorer.score([
PlaceResponse(score=Score(40, 0)),
PlaceResponse(score=Score(50, 0))
])
# should pick one with higher utilization score
assert_that(response, is_(PlaceResponse(score=Score(50, 0))))
def test_same_score_pick_random(self):
responseA = PlaceResponse(score=Score(98, 100))
responseB = PlaceResponse(score=Score(98, 100))
a_freq = 0
b_freq = 0
# Still can fail with 1/2^30 probability, should be negligible
for i in xrange(0, 30):
response = self.scorer.score([responseA, responseB])
if response is responseA:
a_freq += 1
else:
b_freq += 1
assert_that(a_freq, greater_than(0))
assert_that(b_freq, greater_than(0))
@parameterized.expand([
([Score(60, 0), Score(61, 0), Score(62, 0)], 1, [Score(62, 0)]),
([Score(50, 90), Score(50, 90), Score(49, 90)], 1, [Score(50, 90)]),
([Score(60, 0), Score(70, 0), Score(69, 0)], 1, [Score(69, 0),
Score(70, 0)]),
([Score(50, 90), Score(45, 80), Score(10, 70)], 9, [Score(50, 90),
Score(45, 80)]),
])
def test_random_scorer_close_scores(self, scores, ut_ratio, expected):
scorer = RandomScorer(ut_ratio=ut_ratio)
place_responses = [PlaceResponse(score=score) for score in scores]
response = scorer.score(place_responses)
assert_that(expected, has_item(response.score))
