def test_resource_constraints_zero_constraints_min_fanout(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
result = strategy.filter_child(
[self.child_1, self.child_2, self.child_3], self.request, [])
assert_that(result, has_length(2))
def test_resource_constraints_negative_two_select_one(self):
# Test that child 4 is the only one picked as
# child 5 has only one host ("host4") which is included
# in the negative constraints
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1'], True),
ResourceConstraint(ResourceConstraintType.HOST,
['host4'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(1))
assert_that(result, contains_inanyorder(self.child_4))
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1', 'host4'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(1))
assert_that(result, contains_inanyorder(self.child_4))
def test_resource_constraints_negative_two_select_one(self):
# Test that child 4 is the only one picked as
# child 5 has only one host ("host4") which is included
# in the negative constraints
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1'], True),
ResourceConstraint(ResourceConstraintType.HOST,
['host4'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(1))
assert_that(result, contains_inanyorder(self.child_4))
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1', 'host4'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(1))
assert_that(result, contains_inanyorder(self.child_4))
def test_resource_constraints_negative_two_select_two(self):
# Test that both children are picked as
# child 4 has more than just "host1"
# while child 5 has "host4" which is
# not included in the negative constraints
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1'], True),
ResourceConstraint(ResourceConstraintType.HOST,
['host7'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_4, self.child_5))
# Now try with a single constraint
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1', 'host7'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_4, self.child_5))
def test_resource_constraints_zero_constraints_min_fanout(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
result = strategy.filter_child(
[self.child_1, self.child_2, self.child_3], self.request, [])
assert_that(result, has_length(2))
def test_resource_constraints_negative_two_select_two(self):
# Test that both children are picked as
# child 4 has more than just "host1"
# while child 5 has "host4" which is
# not included in the negative constraints
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1'], True),
ResourceConstraint(ResourceConstraintType.HOST,
['host7'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_4, self.child_5))
# Now try with a single constraint
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host1', 'host7'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_4, self.child_5))
def test_resource_constraints_no_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
result = strategy.filter_child(
[self.child_1, self.child_2, self.child_3],
self.request,
[ResourceConstraint(ResourceConstraintType.DATASTORE,
['never_found'])])
assert_that(result, has_length(0))
def test_resource_constraints_no_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
result = strategy.filter_child(
[self.child_1, self.child_2, self.child_3],
self.request,
[ResourceConstraint(ResourceConstraintType.DATASTORE,
['never_found'])])
assert_that(result, has_length(0))
def test_resource_constraints_one_constraints(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
result = strategy.filter_child(
[self.child_1, self.child_2, self.child_3],
self.request,
[ResourceConstraint(ResourceConstraintType.DATASTORE,
['datastore1'])])
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_1, self.child_2))
def test_resource_constraints_negative_all_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host6', 'host7'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_4, self.child_5))
def test_resource_constraints_negative_no_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(
ResourceConstraintType.HOST,
['host1', 'host2', 'host3', 'host4'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(0))
def test_resource_constraints_one_constraints(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
result = strategy.filter_child(
[self.child_1, self.child_2, self.child_3],
self.request,
[ResourceConstraint(ResourceConstraintType.DATASTORE,
['datastore1'])])
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_1, self.child_2))
def test_resource_constraints_negative_all_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(ResourceConstraintType.HOST,
['host6', 'host7'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(2))
assert_that(result, contains_inanyorder(self.child_4, self.child_5))
def test_resource_constraints_negative_no_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [ResourceConstraint(
ResourceConstraintType.HOST,
['host1', 'host2', 'host3', 'host4'], True)]
result = strategy.filter_child(
[self.child_4, self.child_5],
self.request, constraints)
assert_that(result, has_length(0))
def test_resource_constraints_availability_zone_no_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [
ResourceConstraint(ResourceConstraintType.AVAILABILITY_ZONE,
['zone1'], False),
ResourceConstraint(ResourceConstraintType.DATASTORE,
['datastore2'], False)]
result = strategy.filter_child(
[self.child_1, self.child_6, self.child_7],
self.request, constraints)
assert_that(result, has_length(0))
def test_resource_constraints_availability_zone_no_match(self):
strategy = RandomSubsetStrategy(0.5, 2)
strategy._get_constraints = self._get_constraints
constraints = [
ResourceConstraint(ResourceConstraintType.AVAILABILITY_ZONE,
['zone1'], False),
ResourceConstraint(ResourceConstraintType.DATASTORE,
['datastore2'], False)]
result = strategy.filter_child(
[self.child_1, self.child_6, self.child_7],
self.request, constraints)
assert_that(result, has_length(0))
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def get_placement_strategy(self, root_config):
"""
For now this method only accepts root configuration.
This can later be extended to get config for branch and leaf
schedulers as well.
"""
return RandomSubsetStrategy(root_config["fanout_ratio"],
root_config["min_fanout"],
root_config["max_fanout"])
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new leaf scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
:type enable_health_checker: enables health checking of children.
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating leaf scheduler: %s" % scheduler_id)
self.lock = threading.RLock()
self._latch = CountUpDownLatch()
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT,
MAX_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._hosts = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._initialize_services(scheduler_id)
self._health_checker = None
self._enable_health_checker = enable_health_checker
self._configured = ConfigStates.UNINITIALIZED
class BranchScheduler(BaseScheduler):
"""Branch scheduler manages child schedulers."""
# TODO(vspivak): introduce a dynamic timeout depending on where we
# are in the tree. For now assume only 3 levels, making the branch
# schedulers always at level 2.
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def _initialize_services(self, scheduler_id):
""" initializes all the services required by this
scheduler. This allows any test classes to override
the services.
"""
self._threadpool = common.services.get(ThreadPoolExecutor)
self._scheduler_client = SchedulerClient()
def configure(self, schedulers):
"""Configure the branch scheduler.
:param schedulers: list of child scheduler ids
:type schedulers: list of ChildInfo
"""
# Transfer children's constraints from list to set, so searching
# elements are more efficient.
self._schedulers = []
for scheduler in schedulers:
self._schedulers.append(ChildInfo.from_thrift(scheduler))
self._coalesce_resources(self._schedulers)
def find(self, request):
"""Find the specified resource.
:type request: FindRequest
:rtype: FindResponse
"""
futures = []
for scheduler in self._schedulers:
future = self._threadpool.submit(self._find_worker,
scheduler.address, scheduler.port,
scheduler.id,
copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(futures, timeout=FIND_TIMEOUT)
self._logger.info("Find responses received: %d, timed out: %d",
len(done), len(not_done))
for future in done:
response = future.result()
if response.result == FindResultCode.OK:
return response
return FindResponse(FindResultCode.NOT_FOUND)
def place(self, request):
"""Place the specified resources.
:type request: PlaceRequest
:rtype: PlaceResponse
"""
constraints = self._collect_constraints(request.resource)
self._logger.info("Constraints: %s", constraints)
selected = self._placement_schedulers(request, constraints)
if len(selected) == 0:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
done = self._execute_placement(selected, request)
responses = []
had_resource_constraint = False
for future in done:
response = future.result()
if response.result == PlaceResultCode.OK:
responses.append(response)
elif response.result == PlaceResultCode.RESOURCE_CONSTRAINT:
had_resource_constraint = True
best_response = self._scorer.score(responses)
if best_response is not None:
return best_response
elif had_resource_constraint:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
else:
return PlaceResponse(PlaceResultCode.SYSTEM_ERROR)
def _execute_placement(self, schedulers, request):
futures = []
for scheduler in schedulers:
future = self._threadpool.submit(self._place_worker,
scheduler.address, scheduler.port,
scheduler.id,
copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(futures,
timeout=INITIAL_PLACE_TIMEOUT)
self._logger.info(
"Initial place responses received: %d, "
"timed out: %d", len(done), len(not_done))
if len(done) < MIN_PLACE_FAN_OUT:
self._logger.info("Waiting for more place responses")
done, not_done = concurrent.futures.wait(futures,
timeout=PLACE_TIMEOUT -
INITIAL_PLACE_TIMEOUT)
self._logger.info(
"Total place responses received: %d, "
"timed out: %d", len(done), len(not_done))
return done
def _placement_schedulers(self, request, constraints):
return self._place_strategy.filter_child(self._schedulers, request,
constraints)
def _find_worker(self, address, port, scheduler_id, request):
"""Invokes Scheduler.find on a single scheduler.
:type address: str
:type port: str
:type scheduler_id: str
:type request: FindRequest
:rtype: FindResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(address,
port,
scheduler_id,
client_timeout=FIND_TIMEOUT) as client:
return client.find(request)
def _place_worker(self, address, port, scheduler_id, request):
"""Invokes Host.find on a single agent.
:type address: str
:type port: int
:type scheduler_id: str
:type request: PlaceRequest
:rtype: PlaceResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(scheduler_id,
address,
port,
client_timeout=PLACE_TIMEOUT) as client:
return client.place(request)
def cleanup(self):
assert ("Not implemented")
class BranchScheduler(BaseScheduler):
"""Branch scheduler manages child schedulers."""
# TODO(vspivak): introduce a dynamic timeout depending on where we
# are in the tree. For now assume only 3 levels, making the branch
# schedulers always at level 2.
def __init__(self, scheduler_id, ut_ratio, enable_health_checker=True):
"""Create a new branch scheduler.
:param scheduler_id: scheduler id
:type scheduler_id: str
"""
self._logger = logging.getLogger(__name__)
self._logger.info("Creating branch scheduler: %s" % scheduler_id)
self._place_strategy = RandomSubsetStrategy(PLACE_FAN_OUT_RATIO,
MIN_PLACE_FAN_OUT)
self._scheduler_id = scheduler_id
self._schedulers = []
self._scorer = DefaultScorer(ut_ratio)
self._threadpool = None
self._scheduler_client = None
self._initialize_services(scheduler_id)
def _initialize_services(self, scheduler_id):
""" initializes all the services required by this
scheduler. This allows any test classes to override
the services.
"""
self._threadpool = common.services.get(ThreadPoolExecutor)
self._scheduler_client = SchedulerClient()
def configure(self, schedulers):
"""Configure the branch scheduler.
:param schedulers: list of child scheduler ids
:type schedulers: list of ChildInfo
"""
# Transfer children's constraints from list to set, so searching
# elements are more efficient.
self._schedulers = []
for scheduler in schedulers:
self._schedulers.append(ChildInfo.from_thrift(scheduler))
self._coalesce_resources(self._schedulers)
def find(self, request):
"""Find the specified resource.
:type request: FindRequest
:rtype: FindResponse
"""
futures = []
for scheduler in self._schedulers:
future = self._threadpool.submit(
self._find_worker, scheduler.address, scheduler.port,
scheduler.id, copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(futures, timeout=FIND_TIMEOUT)
self._logger.info("Find responses received: %d, timed out: %d",
len(done), len(not_done))
for future in done:
response = future.result()
if response.result == FindResultCode.OK:
return response
return FindResponse(FindResultCode.NOT_FOUND)
def place(self, request):
"""Place the specified resources.
:type request: PlaceRequest
:rtype: PlaceResponse
"""
constraints = self._collect_constraints(request.resource)
self._logger.info("Constraints: %s", constraints)
selected = self._placement_schedulers(request, constraints)
if len(selected) == 0:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
done = self._execute_placement(selected, request)
responses = []
had_resource_constraint = False
for future in done:
response = future.result()
if response.result == PlaceResultCode.OK:
responses.append(response)
elif response.result == PlaceResultCode.RESOURCE_CONSTRAINT:
had_resource_constraint = True
best_response = self._scorer.score(responses)
if best_response is not None:
return best_response
elif had_resource_constraint:
return PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
else:
return PlaceResponse(PlaceResultCode.SYSTEM_ERROR)
def _execute_placement(self, schedulers, request):
futures = []
for scheduler in schedulers:
future = self._threadpool.submit(
self._place_worker, scheduler.address, scheduler.port,
scheduler.id, copy.deepcopy(request))
futures.append(future)
done, not_done = concurrent.futures.wait(
futures, timeout=INITIAL_PLACE_TIMEOUT)
self._logger.info("Initial place responses received: %d, "
"timed out: %d", len(done), len(not_done))
if len(done) < MIN_PLACE_FAN_OUT:
self._logger.info("Waiting for more place responses")
done, not_done = concurrent.futures.wait(
futures, timeout=PLACE_TIMEOUT - INITIAL_PLACE_TIMEOUT)
self._logger.info("Total place responses received: %d, "
"timed out: %d", len(done), len(not_done))
return done
def _placement_schedulers(self, request, constraints):
return self._place_strategy.filter_child(self._schedulers,
request,
constraints)
def _find_worker(self, address, port, scheduler_id, request):
"""Invokes Scheduler.find on a single scheduler.
:type address: str
:type port: str
:type scheduler_id: str
:type request: FindRequest
:rtype: FindResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(address, port, scheduler_id,
client_timeout=FIND_TIMEOUT) as client:
return client.find(request)
def _place_worker(self, address, port, scheduler_id, request):
"""Invokes Host.find on a single agent.
:type address: str
:type port: int
:type scheduler_id: str
:type request: PlaceRequest
:rtype: PlaceResponse
"""
request.scheduler_id = scheduler_id
client = self._scheduler_client
with client.connect(scheduler_id, address, port,
client_timeout=PLACE_TIMEOUT) as client:
return client.place(request)
def cleanup(self):
assert("Not implemented")