def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True): """Create a new branch scheduler. :param scheduler_id: scheduler id :type scheduler_id: str """ self._logger = logging.getLogger(__name__) self._logger.info("Creating branch scheduler: %s" % scheduler_id) self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO, MIN_PLACE_FAN_OUT) self._scheduler_id = scheduler_id self._schedulers = [] self._scorer = DefaultScorer(ut_ratio) self._threadpool = None self._scheduler_client = None self._initialize_services(scheduler_id)
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True): """Create a new branch scheduler. :param scheduler_id: scheduler id :type scheduler_id: str """ self._logger = logging.getLogger(__name__) self._logger.info("Creating branch scheduler: %s" % scheduler_id) self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO, MIN_PLACE_FAN_OUT) self._scheduler_id = scheduler_id self._schedulers = [] self._scorer = DefaultScorer(ut_ratio) self._threadpool = None self._scheduler_client = None self._initialize_services(scheduler_id)
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True): """Create a new leaf scheduler. :param scheduler_id: scheduler id :type scheduler_id: str :type enable_health_checker: enables health checking of children. """ self._logger = logging.getLogger(__name__) self._logger.info("Creating leaf scheduler: %s" % scheduler_id) self.lock = threading.RLock() self._latch = CountUpDownLatch() self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO, MIN_PLACE_FAN_OUT, MAX_PLACE_FAN_OUT) self._scheduler_id = scheduler_id self._hosts = [] self._scorer = DefaultScorer(ut_ratio) self._threadpool = None self._initialize_services(scheduler_id) self._health_checker = None self._enable_health_checker = enable_health_checker self._configured = ConfigStates.UNINITIALIZED
class BranchScheduler(BaseScheduler): """Branch scheduler manages child schedulers.""" # TODO(vspivak): introduce a dynamic timeout depending on where we # are in the tree. For now assume only 3 levels, making the branch # schedulers always at level 2. def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True): """Create a new branch scheduler. :param scheduler_id: scheduler id :type scheduler_id: str """ self._logger = logging.getLogger(__name__) self._logger.info("Creating branch scheduler: %s" % scheduler_id) self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO, MIN_PLACE_FAN_OUT) self._scheduler_id = scheduler_id self._schedulers = [] self._scorer = DefaultScorer(ut_ratio) self._threadpool = None self._scheduler_client = None self._initialize_services(scheduler_id) def _initialize_services(self, scheduler_id): """ initializes all the services required by this scheduler. This allows any test classes to override the services. """ self._threadpool = common.services.get(ThreadPoolExecutor) self._scheduler_client = SchedulerClient() def configure(self, schedulers): """Configure the branch scheduler. :param schedulers: list of child scheduler ids :type schedulers: list of ChildInfo """ # Transfer children's constraints from list to set, so searching # elements are more efficient. self._schedulers = [] for scheduler in schedulers: self._schedulers.append(ChildInfo.from_thrift(scheduler)) self._coalesce_resources(self._schedulers) def find(self, request): """Find the specified resource. :type request: FindRequest :rtype: FindResponse """ futures = [] for scheduler in self._schedulers: future = self._threadpool.submit( self._find_worker, scheduler.address, scheduler.port, scheduler.id, copy.deepcopy(request)) futures.append(future) done, not_done = concurrent.futures.wait(futures, timeout=FIND_TIMEOUT) self._logger.info("Find responses received: %d, timed out: %d", len(done), len(not_done)) for future in done: response = future.result() if response.result == FindResultCode.OK: return response return FindResponse(FindResultCode.NOT_FOUND) def place(self, request): """Place the specified resources. :type request: PlaceRequest :rtype: PlaceResponse """ constraints = self._collect_constraints(request.resource) self._logger.info("Constraints: %s", constraints) selected = self._placement_schedulers(request, constraints) if len(selected) == 0: return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT) done = self._execute_placement(selected, request) responses = [] had_resource_constraint = False for future in done: response = future.result() if response.result == PlaceResultCode.OK: responses.append(response) elif response.result == PlaceResultCode.RESOURCE_CONSTRAINT: had_resource_constraint = True best_response = self._scorer.score(responses) if best_response is not None: return best_response elif had_resource_constraint: return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT) else: return PlaceResponse(PlaceResultCode.SYSTEM_ERROR) def _execute_placement(self, schedulers, request): futures = [] for scheduler in schedulers: future = self._threadpool.submit( self._place_worker, scheduler.address, scheduler.port, scheduler.id, copy.deepcopy(request)) futures.append(future) done, not_done = concurrent.futures.wait( futures, timeout=INITIAL_PLACE_TIMEOUT) self._logger.info("Initial place responses received: %d, " "timed out: %d", len(done), len(not_done)) if len(done) < MIN_PLACE_FAN_OUT: self._logger.info("Waiting for more place responses") done, not_done = concurrent.futures.wait( futures, timeout=PLACE_TIMEOUT - INITIAL_PLACE_TIMEOUT) self._logger.info("Total place responses received: %d, " "timed out: %d", len(done), len(not_done)) return done def _placement_schedulers(self, request, constraints): return self._place_strategy.filter_child(self._schedulers, request, constraints) def _find_worker(self, address, port, scheduler_id, request): """Invokes Scheduler.find on a single scheduler. :type address: str :type port: str :type scheduler_id: str :type request: FindRequest :rtype: FindResponse """ request.scheduler_id = scheduler_id client = self._scheduler_client with client.connect(address, port, scheduler_id, client_timeout=FIND_TIMEOUT) as client: return client.find(request) def _place_worker(self, address, port, scheduler_id, request): """Invokes Host.find on a single agent. :type address: str :type port: int :type scheduler_id: str :type request: PlaceRequest :rtype: PlaceResponse """ request.scheduler_id = scheduler_id client = self._scheduler_client with client.connect(scheduler_id, address, port, client_timeout=PLACE_TIMEOUT) as client: return client.place(request) def cleanup(self): assert("Not implemented")
def test_ratio(self, scores, ratio, expected): scorer = DefaultScorer(ut_ratio=ratio) place_responses = [PlaceResponse(score=score) for score in scores] response = scorer.score(place_responses) assert_that(response.score, is_(expected))
def setUp(self): self.scorer = DefaultScorer(ut_ratio=9)
class ScorerTestCase(unittest.TestCase): def setUp(self): self.scorer = DefaultScorer(ut_ratio=9) def test_highest_score_random(self): place_responses = [] for i in xrange(10): resp = PlaceResponse( score=Score(random.randint(0, 100), random.randint(0, 100))) place_responses.append(resp) self._test_highest_score(place_responses) @parameterized.expand([ ([Score(40, 90), Score(49, 78), Score(50, 70)], 9, Score(50, 70)), ([Score(40, 90), Score(49, 80), Score(50, 70)], 9, Score(49, 80)), ([Score(40, 90), Score(45, 80), Score(50, 70)], 1, Score(40, 90)), ([Score(40, 90), Score(45, 86), Score(50, 70)], 1, Score(45, 86)), ([Score(40, 60), Score(59, 69), Score(50, 70)], 0.1, Score(50, 70)), ([Score(40, 60), Score(61, 69), Score(50, 70)], 0.1, Score(61, 69)), ]) def test_ratio(self, scores, ratio, expected): scorer = DefaultScorer(ut_ratio=ratio) place_responses = [PlaceResponse(score=score) for score in scores] response = scorer.score(place_responses) assert_that(response.score, is_(expected)) def _test_highest_score(self, place_responses): response = self.scorer.score(place_responses) scores = [ self.scorer.score_formula(resp.score) for resp in place_responses ] assert_that(self.scorer.score_formula(response.score), is_(max(scores))) def test_new_resource_score(self): response = self.scorer.score([ PlaceResponse(score=Score(40, 0)), PlaceResponse(score=Score(50, 0)) ]) # should pick one with higher utilization score assert_that(response, is_(PlaceResponse(score=Score(50, 0)))) def test_same_score_pick_random(self): responseA = PlaceResponse(score=Score(98, 100)) responseB = PlaceResponse(score=Score(98, 100)) a_freq = 0 b_freq = 0 # Still can fail with 1/2^30 probability, should be negligible for i in xrange(0, 30): response = self.scorer.score([responseA, responseB]) if response is responseA: a_freq += 1 else: b_freq += 1 assert_that(a_freq, greater_than(0)) assert_that(b_freq, greater_than(0)) @parameterized.expand([ ([Score(60, 0), Score(61, 0), Score(62, 0)], 1, [Score(62, 0)]), ([Score(50, 90), Score(50, 90), Score(49, 90)], 1, [Score(50, 90)]), ([Score(60, 0), Score(70, 0), Score(69, 0)], 1, [Score(69, 0), Score(70, 0)]), ([Score(50, 90), Score(45, 80), Score(10, 70)], 9, [Score(50, 90), Score(45, 80)]), ]) def test_random_scorer_close_scores(self, scores, ut_ratio, expected): scorer = RandomScorer(ut_ratio=ut_ratio) place_responses = [PlaceResponse(score=score) for score in scores] response = scorer.score(place_responses) assert_that(expected, has_item(response.score))
class BranchScheduler(BaseScheduler): """Branch scheduler manages child schedulers.""" # TODO(vspivak): introduce a dynamic timeout depending on where we # are in the tree. For now assume only 3 levels, making the branch # schedulers always at level 2. def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True): """Create a new branch scheduler. :param scheduler_id: scheduler id :type scheduler_id: str """ self._logger = logging.getLogger(__name__) self._logger.info("Creating branch scheduler: %s" % scheduler_id) self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO, MIN_PLACE_FAN_OUT) self._scheduler_id = scheduler_id self._schedulers = [] self._scorer = DefaultScorer(ut_ratio) self._threadpool = None self._scheduler_client = None self._initialize_services(scheduler_id) def _initialize_services(self, scheduler_id): """ initializes all the services required by this scheduler. This allows any test classes to override the services. """ self._threadpool = common.services.get(ThreadPoolExecutor) self._scheduler_client = SchedulerClient() def configure(self, schedulers): """Configure the branch scheduler. :param schedulers: list of child scheduler ids :type schedulers: list of ChildInfo """ # Transfer children's constraints from list to set, so searching # elements are more efficient. self._schedulers = [] for scheduler in schedulers: self._schedulers.append(ChildInfo.from_thrift(scheduler)) self._coalesce_resources(self._schedulers) def find(self, request): """Find the specified resource. :type request: FindRequest :rtype: FindResponse """ futures = [] for scheduler in self._schedulers: future = self._threadpool.submit(self._find_worker, scheduler.address, scheduler.port, scheduler.id, copy.deepcopy(request)) futures.append(future) done, not_done = concurrent.futures.wait(futures, timeout=FIND_TIMEOUT) self._logger.info("Find responses received: %d, timed out: %d", len(done), len(not_done)) for future in done: response = future.result() if response.result == FindResultCode.OK: return response return FindResponse(FindResultCode.NOT_FOUND) def place(self, request): """Place the specified resources. :type request: PlaceRequest :rtype: PlaceResponse """ constraints = self._collect_constraints(request.resource) self._logger.info("Constraints: %s", constraints) selected = self._placement_schedulers(request, constraints) if len(selected) == 0: return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT) done = self._execute_placement(selected, request) responses = [] had_resource_constraint = False for future in done: response = future.result() if response.result == PlaceResultCode.OK: responses.append(response) elif response.result == PlaceResultCode.RESOURCE_CONSTRAINT: had_resource_constraint = True best_response = self._scorer.score(responses) if best_response is not None: return best_response elif had_resource_constraint: return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT) else: return PlaceResponse(PlaceResultCode.SYSTEM_ERROR) def _execute_placement(self, schedulers, request): futures = [] for scheduler in schedulers: future = self._threadpool.submit(self._place_worker, scheduler.address, scheduler.port, scheduler.id, copy.deepcopy(request)) futures.append(future) done, not_done = concurrent.futures.wait(futures, timeout=INITIAL_PLACE_TIMEOUT) self._logger.info( "Initial place responses received: %d, " "timed out: %d", len(done), len(not_done)) if len(done) < MIN_PLACE_FAN_OUT: self._logger.info("Waiting for more place responses") done, not_done = concurrent.futures.wait(futures, timeout=PLACE_TIMEOUT - INITIAL_PLACE_TIMEOUT) self._logger.info( "Total place responses received: %d, " "timed out: %d", len(done), len(not_done)) return done def _placement_schedulers(self, request, constraints): return self._place_strategy.filter_child(self._schedulers, request, constraints) def _find_worker(self, address, port, scheduler_id, request): """Invokes Scheduler.find on a single scheduler. :type address: str :type port: str :type scheduler_id: str :type request: FindRequest :rtype: FindResponse """ request.scheduler_id = scheduler_id client = self._scheduler_client with client.connect(address, port, scheduler_id, client_timeout=FIND_TIMEOUT) as client: return client.find(request) def _place_worker(self, address, port, scheduler_id, request): """Invokes Host.find on a single agent. :type address: str :type port: int :type scheduler_id: str :type request: PlaceRequest :rtype: PlaceResponse """ request.scheduler_id = scheduler_id client = self._scheduler_client with client.connect(scheduler_id, address, port, client_timeout=PLACE_TIMEOUT) as client: return client.place(request) def cleanup(self): assert ("Not implemented")
def test_ratio(self, scores, ratio, expected): scorer = DefaultScorer(ut_ratio=ratio) place_responses = [PlaceResponse(score=score) for score in scores] response = scorer.score(place_responses) assert_that(response.score, is_(expected))
def setUp(self): self.scorer = DefaultScorer(ut_ratio=9)
class ScorerTestCase(unittest.TestCase): def setUp(self): self.scorer = DefaultScorer(ut_ratio=9) def test_highest_score_random(self): place_responses = [] for i in xrange(10): resp = PlaceResponse(score=Score(random.randint(0, 100), random.randint(0, 100))) place_responses.append(resp) self._test_highest_score(place_responses) @parameterized.expand([ ([Score(40, 90), Score(49, 78), Score(50, 70)], 9, Score(50, 70)), ([Score(40, 90), Score(49, 80), Score(50, 70)], 9, Score(49, 80)), ([Score(40, 90), Score(45, 80), Score(50, 70)], 1, Score(40, 90)), ([Score(40, 90), Score(45, 86), Score(50, 70)], 1, Score(45, 86)), ([Score(40, 60), Score(59, 69), Score(50, 70)], 0.1, Score(50, 70)), ([Score(40, 60), Score(61, 69), Score(50, 70)], 0.1, Score(61, 69)), ]) def test_ratio(self, scores, ratio, expected): scorer = DefaultScorer(ut_ratio=ratio) place_responses = [PlaceResponse(score=score) for score in scores] response = scorer.score(place_responses) assert_that(response.score, is_(expected)) def _test_highest_score(self, place_responses): response = self.scorer.score(place_responses) scores = [self.scorer.score_formula(resp.score) for resp in place_responses] assert_that(self.scorer.score_formula(response.score), is_(max( scores))) def test_new_resource_score(self): response = self.scorer.score([ PlaceResponse(score=Score(40, 0)), PlaceResponse(score=Score(50, 0)) ]) # should pick one with higher utilization score assert_that(response, is_(PlaceResponse(score=Score(50, 0)))) def test_same_score_pick_random(self): responseA = PlaceResponse(score=Score(98, 100)) responseB = PlaceResponse(score=Score(98, 100)) a_freq = 0 b_freq = 0 # Still can fail with 1/2^30 probability, should be negligible for i in xrange(0, 30): response = self.scorer.score([responseA, responseB]) if response is responseA: a_freq += 1 else: b_freq += 1 assert_that(a_freq, greater_than(0)) assert_that(b_freq, greater_than(0)) @parameterized.expand([ ([Score(60, 0), Score(61, 0), Score(62, 0)], 1, [Score(62, 0)]), ([Score(50, 90), Score(50, 90), Score(49, 90)], 1, [Score(50, 90)]), ([Score(60, 0), Score(70, 0), Score(69, 0)], 1, [Score(69, 0), Score(70, 0)]), ([Score(50, 90), Score(45, 80), Score(10, 70)], 9, [Score(50, 90), Score(45, 80)]), ]) def test_random_scorer_close_scores(self, scores, ut_ratio, expected): scorer = RandomScorer(ut_ratio=ut_ratio) place_responses = [PlaceResponse(score=score) for score in scores] response = scorer.score(place_responses) assert_that(expected, has_item(response.score))