def _create_next_tasks(self, when):
"""Creates the next tasks that needs to be run for this node. Caller must have obtained the thread lock.
:param when: The current time
:type when: :class:`datetime.datetime`
"""
# If we have a cleanup task, check that node's agent ID has not changed
if self._cleanup_task and self._cleanup_task.agent_id != self._agent_id:
self._cleanup_task = None
if not self._cleanup_task and self._is_ready_for_cleanup_task(when):
self._cleanup_task = self._cleanup.create_next_task(
self._agent_id, self._hostname,
self._is_initial_cleanup_completed)
# If we have a health task, check that node's agent ID has not changed
if self._health_task and self._health_task.agent_id != self._agent_id:
self._health_task = None
if not self._health_task and self._is_ready_for_health_task(when):
self._health_task = HealthTask(scheduler_mgr.framework_id,
self._agent_id)
# If we have a pull task, check that node's agent ID has not changed
if self._pull_task and self._pull_task.agent_id != self._agent_id:
self._pull_task = None
if not self._pull_task and self._is_ready_for_pull_task(when):
self._pull_task = PullTask(scheduler_mgr.framework_id,
self._agent_id)
def test_accept_node_tasks_insufficient_resources(self):
"""Tests calling accept_node_tasks() when there are not enough resources"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
health_task = HealthTask('1234', 'agent_1')
pull_task = PullTask('1234', 'agent_1')
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
node.get_next_tasks = MagicMock()
node.get_next_tasks.return_value = [health_task, pull_task]
offered_resources = NodeResources([Cpus(0.0), Mem(50.0)])
task_resources = NodeResources()
watermark_resources = NodeResources([Cpus(100.0), Mem(500.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
waiting_tasks = []
had_waiting_task = scheduling_node.accept_node_tasks(
now(), waiting_tasks)
self.assertTrue(had_waiting_task)
self.assertEqual(len(scheduling_node.allocated_tasks), 0)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(NodeResources()))
self.assertTrue(
scheduling_node._remaining_resources.is_equal(offered_resources))
self.assertListEqual(waiting_tasks, [health_task, pull_task])
def test_accept_node_tasks(self):
"""Tests successfully calling accept_node_tasks()"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
health_task = HealthTask('1234', 'agent_1')
pull_task = PullTask('1234', 'agent_1')
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
node.get_next_tasks = MagicMock()
node.get_next_tasks.return_value = [health_task, pull_task]
node_task_resources = NodeResources()
node_task_resources.add(health_task.get_resources())
node_task_resources.add(pull_task.get_resources())
offered_resources = NodeResources([Cpus(100.0), Mem(5000.0)])
expected_remaining_resources = NodeResources()
expected_remaining_resources.add(offered_resources)
expected_remaining_resources.subtract(node_task_resources)
task_resources = NodeResources()
watermark_resources = NodeResources([Cpus(100.0), Mem(5000.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
waiting_tasks = []
had_waiting_task = scheduling_node.accept_node_tasks(
now(), waiting_tasks)
self.assertFalse(had_waiting_task)
self.assertEqual(len(scheduling_node.allocated_tasks), 2)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(node_task_resources))
self.assertTrue(
scheduling_node._remaining_resources.is_equal(
expected_remaining_resources))
self.assertListEqual(waiting_tasks, [])
def test_add_allocated_offers_remove_all_tasks(self):
"""Tests calling add_allocated_offers() when there are not enough resources for the job exes or node tasks"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
health_task = HealthTask('1234', 'agent_1')
pull_task = PullTask('1234', 'agent_1')
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
node.get_next_tasks = MagicMock()
node.get_next_tasks.return_value = [health_task, pull_task]
offered_resources = NodeResources([Cpus(100.0), Mem(500.0)])
watermark_resources = NodeResources([Cpus(100.0), Mem(500.0)])
resource_set = ResourceSet(offered_resources, NodeResources(),
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
running_job_exe_1 = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(1.0), Mem(10.0)]))
running_job_exe_2 = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(2.0), Mem(20.0)]))
node_task_resources = NodeResources()
node_task_resources.add(health_task.get_resources())
node_task_resources.add(pull_task.get_resources())
all_required_resources = NodeResources()
all_required_resources.add(node_task_resources)
all_required_resources.add(
running_job_exe_1.next_task().get_resources())
all_required_resources.add(
running_job_exe_2.next_task().get_resources())
expected_remaining_resources = NodeResources()
expected_remaining_resources.add(offered_resources)
expected_remaining_resources.subtract(node_task_resources)
# Set up node with node tasks and job exes (there would never be queued job exes since they would be scheduled
# before add_allocated_offers() was called
scheduling_node.accept_node_tasks(now(), [])
scheduling_node.accept_job_exe_next_task(running_job_exe_1, [])
scheduling_node.accept_job_exe_next_task(running_job_exe_2, [])
self.assertEqual(len(scheduling_node.allocated_tasks), 2)
self.assertEqual(len(scheduling_node._allocated_running_job_exes), 2)
self.assertEqual(len(scheduling_node._allocated_queued_job_exes), 0)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(
all_required_resources))
# Set up offers (not enough for job exes or node tasks)
offer_1 = ResourceOffer('offer_1', 'agent_1', '1234',
NodeResources([Cpus(0.1),
Mem(600.0)]), now(), None)
scheduling_node.add_allocated_offers([offer_1])
self.assertListEqual(scheduling_node.allocated_offers, [offer_1])
# All allocated tasks and job exes should be gone
self.assertEqual(len(scheduling_node.allocated_tasks), 0)
self.assertEqual(len(scheduling_node._allocated_running_job_exes), 0)
self.assertEqual(len(scheduling_node._allocated_queued_job_exes), 0)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(NodeResources()))
self.assertTrue(
scheduling_node._remaining_resources.is_equal(offered_resources))
