def _build_scheduler_configure(self):
child_1 = ChildInfo(
id="baz_1", address="baz_1", port=1024,
constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_1"]),
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_2"]),
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_3"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_1"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_2"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_3"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_1"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_2"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_3"])
])
child_2 = ChildInfo(
id="baz_2", address="baz_2", port=1024,
constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_4"]),
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_5"]),
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_6"]),
# duplicate on purpose
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_6"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_4"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_5"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_6"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_4"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_5"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_6"])
])
child_3 = ChildInfo(
id="baz_3", address="baz_3", port=1024,
constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE,
["ds_7", "ds_8", "ds_9"]),
ResourceConstraint(ResourceConstraintType.DATASTORE_TAG,
["ds_tag_7", "ds_tag_8", "ds_tag_9"]),
ResourceConstraint(ResourceConstraintType.NETWORK,
["net_7", "net_8", "net_9"])])
return [child_1, child_2, child_3]
def get_schedulers(self, request):
"""Return the list of current schedulers.
:type request: GetSchedulersRequest:
:rtype: GetSchedulersResponse
"""
scheduler_handler = common.services.get(Scheduler.Iface)
_schedulers = scheduler_handler.get_schedulers
response = GetSchedulersResponse()
response.schedulers = []
for schId in _schedulers:
scheduler = _schedulers[schId]
schEntry = SchedulerEntry()
schRole = SchedulerRole()
schRole.host_children = []
schRole.id = schId
for host in scheduler._get_hosts():
childHost = ChildInfo()
childHost.id = host.id
childHost.address = host.address
childHost.port = host.port
schRole.host_children.append(childHost)
schEntry.role = schRole
response.schedulers.append(schEntry)
response.result = GetSchedulersResultCode.OK
return response
def test_place_with_resource_constraints(self, client_class):
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
bar_response = PlaceResponse(PlaceResultCode.OK, agent_id="bar",
score=Score(5, 90))
bar_client.place.return_value = bar_response
self._clients["bar"] = bar_client
baz_client = MagicMock()
baz_response = PlaceResponse(PlaceResultCode.OK, agent_id="baz",
score=Score(30, 80))
baz_client.place.return_value = baz_response
self._clients["baz"] = baz_client
scheduler = BranchScheduler("foo", 9)
scheduler.configure([
ChildInfo(id="bar", address="bar", constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE, ["1"])]),
ChildInfo(id="baz", address="baz", constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE, ["2"])])])
request = self._place_request()
request.resource.vm.resource_constraints = [ResourceConstraint(
ResourceConstraintType.DATASTORE, ["1"])]
response = scheduler.place(request)
assert_that(response.result, is_(PlaceResultCode.OK))
assert_that(response.agent_id, is_("bar"))
def test_place_resource_placed(self, client_class):
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
bar_response = PlaceResponse(PlaceResultCode.OK,
agent_id="bar",
score=Score(5, 90))
bar_client.place.return_value = bar_response
self._clients["bar"] = bar_client
baz_client = MagicMock()
baz_response = PlaceResponse(PlaceResultCode.OK,
agent_id="baz",
score=Score(30, 80))
baz_client.place.return_value = baz_response
self._clients["baz"] = baz_client
scheduler = BranchScheduler("foo", 9)
scheduler.configure([
ChildInfo(id="bar", address="bar"),
ChildInfo(id="baz", address="baz")
])
response = scheduler.place(self._place_request())
assert_that(response, is_(same_instance(baz_response)))
def get_schedulers(self, request):
"""Return the list of current schedulers.
:type request: GetSchedulersRequest:
:rtype: GetSchedulersResponse
"""
scheduler_handler = common.services.get(Scheduler.Iface)
_schedulers = scheduler_handler.get_schedulers
response = GetSchedulersResponse()
response.schedulers = []
for schId in _schedulers:
scheduler = _schedulers[schId]
schEntry = SchedulerEntry()
schRole = SchedulerRole()
schRole.host_children = []
schRole.id = schId
for host in scheduler._get_hosts():
childHost = ChildInfo()
childHost.id = host.id
childHost.address = host.address
childHost.port = host.port
schRole.host_children.append(childHost)
schEntry.role = schRole
response.schedulers.append(schEntry)
response.result = GetSchedulersResultCode.OK
return response
def test_place_with_resource_constraints_no_match(self):
scheduler = BranchScheduler("foo", 9)
scheduler.configure([
ChildInfo(id="bar", constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE, ["1"])]),
ChildInfo(id="baz", constraints=[
ResourceConstraint(ResourceConstraintType.DATASTORE, ["2"])])])
request = self._place_request()
request.resource.vm.resource_constraints = [ResourceConstraint(
"datastore", ["never_found"])]
response = scheduler.place(request)
assert_that(response.result, is_(PlaceResultCode.RESOURCE_CONSTRAINT))
def test_get_schedulers(self, leaf_scheduler_cls):
leaf_scheduler = MagicMock()
leaf_scheduler_cls.return_value = leaf_scheduler
common.services.register(Host.Iface, MagicMock())
scheduler_handler = SchedulerHandler()
common.services.register(Scheduler.Iface, scheduler_handler)
agent_config = MagicMock()
agent_config.reboot_required = False
common.services.register(ServiceName.AGENT_CONFIG, agent_config)
common.services.register(ServiceName.REQUEST_ID, MagicMock())
agent_control_handler = AgentControlHandler()
request = GetSchedulersRequest()
response = agent_control_handler.get_schedulers(request)
child_host = ChildInfo(id="foo", address="address", port=12345)
assert_that(response.schedulers, is_(empty()))
leaf_scheduler_id = "leaf1"
scheduler_handler.configure(
[SchedulerRole(leaf_scheduler_id, "parent1",
host_children=[child_host])])
leaf_scheduler._get_hosts.return_value = [child_host]
response = agent_control_handler.get_schedulers(request)
assert_that(response.schedulers, has_length(1))
scheduler = response.schedulers[0]
assert_that(scheduler.role.id, is_(leaf_scheduler_id))
assert_that(scheduler.role.host_children[0], is_(child_host))
assert_that(len(scheduler.role.host_children), is_(1))
assert_that(response.result, is_(GetSchedulersResultCode.OK))
def test_configure_create_leaf_scheduler(self, leaf_scheduler_cls):
leaf_scheduler = MagicMock()
leaf_scheduler_cls.return_value = leaf_scheduler
handler = SchedulerHandler()
assert_that(handler._schedulers, is_(empty()))
handler.configure([
SchedulerRole("leaf-scheduler",
"parent-id",
host_children=[ChildInfo(id="foo")])
])
assert_that(handler._schedulers, has_length(1))
scheduler = handler._schedulers["leaf-scheduler"]
assert_that(scheduler.id, is_(leaf_scheduler.id))
scheduler.configure.assert_called_once_with([ChildInfo(id="foo")])
def test_place_res_with_missing(self, health_checker, client_class):
client_class.side_effect = self.create_fake_client
_health_checker = MagicMock()
health_checker.return_value = _health_checker
_health_checker.get_missing_hosts = ["bar"]
baz_client = MagicMock()
baz_response = PlaceResponse(PlaceResultCode.OK, agent_id="baz",
score=Score(30, 80))
baz_client.host_place.return_value = baz_response
self._clients["baz"] = baz_client
scheduler = LeafScheduler("foo", 1.0, True)
scheduler.configure([ChildInfo(id="bar", address="bar"),
ChildInfo(id="baz", address="baz")])
response = scheduler.place(self._place_request())
assert_that(response, is_(same_instance(baz_response)))
def test_find_resource_found(self, client_class):
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
bar_response = FindResponse(FindResultCode.NOT_FOUND)
bar_client.host_find.return_value = bar_response
self._clients["bar"] = bar_client
baz_client = MagicMock()
baz_response = FindResponse(FindResultCode.OK)
baz_client.host_find.return_value = baz_response
self._clients["baz"] = baz_client
scheduler = LeafScheduler("foo", 9, False)
scheduler.configure([ChildInfo(id="bar", address="bar"),
ChildInfo(id="baz", address="baz")])
response = scheduler.find(self._place_request())
assert_that(response, is_(same_instance(baz_response)))
def test_find_resource_not_found(self, client_class):
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
bar_response = FindResponse(FindResultCode.NOT_FOUND)
bar_client.find.return_value = bar_response
self._clients["bar"] = bar_client
baz_client = MagicMock()
baz_response = FindResponse(FindResultCode.NOT_FOUND)
baz_client.find.return_value = baz_response
self._clients["baz"] = baz_client
scheduler = BranchScheduler("foo", 9)
scheduler.configure([ChildInfo(id="bar", address="bar"),
ChildInfo(id="baz", address="baz")])
response = scheduler.find(FindRequest())
assert_that(response.result, is_(FindResultCode.NOT_FOUND))
def test_place_resource_not_found(self, client_class):
client_class.side_effect = self.create_fake_client
bar_client = MagicMock()
bar_response = PlaceResponse(PlaceResultCode.RESOURCE_CONSTRAINT)
bar_client.place.return_value = bar_response
self._clients["bar"] = bar_client
baz_client = MagicMock()
baz_response = PlaceResponse(PlaceResultCode.SYSTEM_ERROR)
baz_client.place.return_value = baz_response
self._clients["baz"] = baz_client
scheduler = BranchScheduler("foo", 9)
scheduler.configure([ChildInfo(id="bar", address="bar"),
ChildInfo(id="baz", address="baz")])
response = scheduler.place(self._place_request())
assert_that(response.result, is_(PlaceResultCode.RESOURCE_CONSTRAINT))
def test_find_sets_scheduler_id(self, client_class):
client_class.side_effect = self.create_fake_client
client = MagicMock()
self._clients["bar"] = client
scheduler = BranchScheduler("foo", 9)
scheduler.configure([ChildInfo(id="bar", address="bar")])
scheduler.find(FindRequest())
client.find.assert_called_with(FindRequest(scheduler_id="bar"))
def test_place_clears_scheduler_id(self, client_class):
client_class.side_effect = self.create_fake_client
client = MagicMock()
self._clients["bar"] = client
scheduler = LeafScheduler("foo", 9, False)
scheduler.configure([ChildInfo(id="bar", address="bar")])
scheduler.place(PlaceRequest(scheduler_id="foo"))
client.host_place.assert_called_with(PlaceRequest())
def test_configure_update_scheduler(self):
handler = SchedulerHandler()
handler.configure([
SchedulerRole("leaf-scheduler",
"parent-id",
host_children=[ChildInfo(id="foo")])
], False)
old_scheduler = handler._schedulers["leaf-scheduler"]
handler.configure([
SchedulerRole("leaf-scheduler",
"parent-id",
host_children=[ChildInfo(id="bar")])
], False)
new_scheduler = handler._schedulers["leaf-scheduler"]
assert_that(new_scheduler._scheduler_id,
equal_to(old_scheduler._scheduler_id))
assert_that(new_scheduler._hosts, has_length(1))
assert_that(new_scheduler._hosts[0].id, equal_to("bar"))
def test_place_during_reconfigure(self):
# Create leaf scheduler which will return RESOURCE_CONSTRAINT
scheduler = LeafScheduler("foo", 1, False)
scheduler._placement_hosts = MagicMock()
scheduler._placement_hosts.return_value = []
# Create scheduler handler
handler = SchedulerHandler()
handler._create_scheduler = MagicMock()
handler._create_scheduler.return_value = scheduler
# Configure scheduler handler with leaf scheduler
handler.configure([
SchedulerRole("foo",
"parent-id",
scheduler_children=[ChildInfo(id="child")])
])
assert_that(handler._schedulers, has_length(1))
concurrency = 5
threads = []
results = {}
done = [False]
# Define the thread which keeps calling place, to make sure it
# always return RESOURCE_CONSTRAINT
def _loop():
while True:
actual_response = handler.place(PlaceRequest(
Resource(), "foo"))
assert_that(actual_response.result,
is_not(PlaceResultCode.SYSTEM_ERROR))
if done[0]:
results[threading.current_thread().name] = True
break
for _ in xrange(concurrency):
thread = threading.Thread(target=_loop)
thread.start()
threads.append(thread)
# Reconfigure scheduler to demote it. To test when reconfiguring,
# there is no race when new schedulers replacing the old ones.
handler.configure([])
assert_that(handler._schedulers, has_length(0))
time.sleep(0.1)
done[0] = True
for thread in threads:
thread.join()
assert_that(len(results), equal_to(concurrency))
def test_get_leaf_scheduler(self):
"""Test agent introspection"""
agent_host = 'localhost'
agent_port = 20000
# Agent not online
leaf = get_leaf_scheduler(agent_host, agent_port)
assert_that(leaf, is_(None))
# Start an agent with an invalid chairman, so that it doesn't
# get configured, because we want to configure it manually
config = self.runtime.get_agent_config(agent_host, agent_port,
"localhost", 24234)
res = self.runtime.start_agent(config)
agent_client = res[1]
# Agent is online but not a leaf scheduler
leaf = get_leaf_scheduler(agent_host, agent_port)
assert_that(leaf, is_(None))
leafId1 = stable_uuid("leaf scheduler")
config_req = THost.GetConfigRequest()
host_config = agent_client.get_host_config(config_req).hostConfig
leaf_scheduler = SchedulerRole(leafId1)
leaf_scheduler.parent_id = stable_uuid("parent scheduler")
leaf_scheduler.hosts = [host_config.agent_id]
leaf_scheduler.host_children = [
ChildInfo(id=host_config.agent_id,
address=agent_host,
port=agent_port)
]
config_request = ConfigureRequest(leafId1, Roles([leaf_scheduler]))
resp = agent_client.configure(config_request)
assert_that(resp.result, is_(ConfigureResultCode.OK))
leaf = get_leaf_scheduler(agent_host, agent_port)
assert_that(leaf.id, not_none())
assert_that(leaf.type, is_(LEAF_SCHEDULER_TYPE))
assert_that(len(leaf.children), is_(1))
# Verify the owner host
owner_host = leaf.owner
assert_that(owner_host, not_none())
assert_that(owner_host.id, is_(host_config.agent_id))
assert_that(owner_host.address, is_(agent_host))
assert_that(owner_host.port, is_(agent_port))
assert_that(owner_host.parent, is_(leaf))
def test_place_local_agent(self, client_class):
client_class.side_effect = self.create_fake_client
remote_client = MagicMock()
self._clients["local-id"] = remote_client
local_response = PlaceResponse(
PlaceResultCode.OK, agent_id="local-id", score=Score(5, 90))
self._scheduler_handler.host_place.return_value = local_response
scheduler = LeafScheduler("foo", 9, False)
scheduler.configure([ChildInfo(id="local-id")])
response = scheduler.place(self._place_request())
assert_that(remote_client.place.called, is_(False))
assert_that(response, is_(same_instance(local_response)))
def test_configure_cleanup_scheduler(self, leaf_scheduler_cls):
leaf_scheduler = MagicMock()
leaf_scheduler_cls.return_value = leaf_scheduler
handler = SchedulerHandler()
handler.configure([
SchedulerRole("leaf-scheduler",
"parent-id",
host_children=[ChildInfo(id="foo")])
])
scheduler = handler._schedulers["leaf-scheduler"]
assert_that(scheduler, is_(leaf_scheduler))
handler.configure([])
assert_that(scheduler.cleanup.called, is_(True))
def test_place_exception(self, client_class):
client_class.side_effect = self.create_fake_client
client = MagicMock()
e = TimeoutError()
client.place.side_effect = e
self._clients["baz"] = client
scheduler = LeafScheduler("foo", 1.0, False)
scheduler.configure([ChildInfo(id="baz", address="baz", port=123)])
response = scheduler.place(self._place_request())
assert_that(response.result, is_(PlaceResultCode.SYSTEM_ERROR))
# must shutdown to join the futures since the patch lifecycle
# ends when this method returns
self._threadpool.shutdown()
def _create_scheduler_with_hosts(self, name, number_of_hosts):
scheduler = LeafScheduler(name, 1.0, False)
hosts = []
for num in range(number_of_hosts):
child = name + "_host_" + str(num)
child_id = "id_" + child
child_address = "address_" + child
hosts.append(
ChildInfo(id=child_id, address=child_address, port=num))
client = MagicMock()
response = PlaceResponse(PlaceResultCode.OK,
agent_id=child_id,
score=Score(100, 90))
client.host_place.return_value = response
self._clients[child_address] = client
scheduler.configure(hosts)
return scheduler
def test_place_timeout(self, client_class, wait_fn):
client_class.side_effect = self.create_fake_client
client = MagicMock()
response = PlaceResponse(PlaceResultCode.OK)
client.place.return_value = response
self._clients["scheduler/baz"] = client
wait_fn.return_value = set(), set()
scheduler = BranchScheduler("foo", 9)
scheduler.configure([ChildInfo(id="baz")])
response = scheduler.place(self._place_request())
assert_that(response.result, is_(PlaceResultCode.SYSTEM_ERROR))
# must shutdown to join the futures since the patch lifecycle
# ends when this method returns
self._threadpool.shutdown()
def test_find_timeout(self, wait_fn, client_class):
client_class.side_effect = self.create_fake_client
client = MagicMock()
response = FindResponse(FindResultCode.OK)
client.find.return_value = response
self._clients["baz"] = client
scheduler = BranchScheduler("foo", 9)
scheduler.configure([ChildInfo(id="baz", address="baz")])
wait_fn.return_value = set(), set()
response = scheduler.find(FindRequest())
assert_that(response.result, is_(FindResultCode.NOT_FOUND))
# must shutdown to join the futures since the patch lifecycle
# ends when this method returns
self._threadpool.shutdown()
def test_demote_agent_from_leaf_scheduler(self):
request = GetSchedulersRequest()
response = self.control_client.get_schedulers(request)
# Agent starts without any schedulers
assert_that(len(response.schedulers), is_(0))
# Configure the agent with a leaf scheduler
leafId1 = stable_uuid("leaf scheduler")
config_req = Host.GetConfigRequest()
host_config = self.client.get_host_config(config_req).hostConfig
leaf_scheduler = SchedulerRole(leafId1)
leaf_scheduler.parent_id = stable_uuid("parent scheduler")
leaf_scheduler.hosts = [host_config.agent_id]
leaf_scheduler.host_children = [ChildInfo(id=host_config.agent_id,
address="localhost",
port=8835)]
config_request = ConfigureRequest(
leafId1,
Roles([leaf_scheduler]))
self.client.configure(config_request)
request = GetSchedulersRequest()
response = self.control_client.get_schedulers(request)
# Verify that the agent has been configured
assert_that(len(response.schedulers), is_(1))
assert_that(response.schedulers[0].role.id, is_(leafId1))
# Demote agent from leaf scheduler
config_request = ConfigureRequest(
leafId1,
Roles([]))
self.client.configure(config_request)
# Verify that the agent isn't a scheduler
request = GetSchedulersRequest()
response = self.control_client.get_schedulers(request)
assert_that(len(response.schedulers), is_(0))