def test_score_job_exe_for_reservation_insufficient_resources(self):
"""Tests calling score_job_exe_for_reservation() when there are not enough resources to reserve for the job"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(20.0), Mem(100.0)])
watermark_resources = NodeResources([Cpus(200.0), Mem(700.0)])
resource_set = ResourceSet(offered_resources, NodeResources(),
watermark_resources)
task = HealthTask(
'1234', 'agent_1') # Resources are 0.1 CPUs and 32 MiB memory
job_exe_1 = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(10.0), Mem(50.0)]),
priority=1000)
job_exe_2 = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(56.0), Mem(15.0)]),
priority=100)
scheduling_node = SchedulingNode('agent_1', node, [task],
[job_exe_1, job_exe_2], resource_set)
queue_model_1 = queue_test_utils.create_queue(priority=100,
cpus_required=8.0,
mem_required=40.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe_1 = QueuedJobExecution(queue_model_1)
queue_model_2 = queue_test_utils.create_queue(priority=1000,
cpus_required=8.0,
mem_required=40.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe_2 = QueuedJobExecution(queue_model_2)
scheduling_node.accept_new_job_exe(job_exe_1)
scheduling_node.accept_new_job_exe(job_exe_2)
# We are going to try to reserve the node for a job execution with priority 120
# Calculate available resources for reservation:
# Watermark (200, 700) - System Tasks (0.1, 32) - Higher Priority Existing Job Exes (56, 15) - Higher Priority
# New Job Exes (8, 40) = 135.9 CPUs, 613 memory
# This new job should NOT fit for reservation
queue_model = queue_test_utils.create_queue(priority=120,
cpus_required=140.0,
mem_required=600.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe = QueuedJobExecution(queue_model)
job_type_resource_1 = NodeResources([Cpus(2.0), Mem(10.0)])
score = scheduling_node.score_job_exe_for_reservation(
job_exe, [job_type_resource_1])
self.assertIsNone(score)
def test_consider_new_job_exe(self):
"""Tests consider_new_job_exe() and get_accepted_new_job_exes()"""
node_offers = NodeOffers(self.node)
offer_1 = ResourceOffer(
'offer_1', self.node_agent,
NodeResources(cpus=24.0, mem=1024.0, disk=1024.0))
node_offers.add_offer(offer_1)
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=50.0, mem=2048.0, disk=2048.0))
node_offers.add_offer(offer_2)
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_running_job_exes(), [])
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
job_exe_1 = QueuedJobExecution(self.queue_1)
result = node_offers.consider_new_job_exe(job_exe_1)
self.assertEqual(result, NodeOffers.ACCEPTED)
result = node_offers.consider_new_job_exe(
job_exe_1) # Same job_exe, should have no effect
self.assertEqual(result, NodeOffers.ACCEPTED)
job_exe_high_cpus = QueuedJobExecution(self.queue_high_cpus)
result = node_offers.consider_new_job_exe(job_exe_high_cpus)
self.assertEqual(result, NodeOffers.NOT_ENOUGH_CPUS)
job_exe_high_mem = QueuedJobExecution(self.queue_high_mem)
result = node_offers.consider_new_job_exe(job_exe_high_mem)
self.assertEqual(result, NodeOffers.NOT_ENOUGH_MEM)
job_exe_high_disk = QueuedJobExecution(self.queue_high_disk)
result = node_offers.consider_new_job_exe(job_exe_high_disk)
self.assertEqual(result, NodeOffers.NOT_ENOUGH_DISK)
job_exe_2 = QueuedJobExecution(self.queue_2)
result = node_offers.consider_new_job_exe(job_exe_2)
self.assertEqual(result, NodeOffers.ACCEPTED)
self.assertTrue(node_offers.has_accepted_job_exes())
self.assertEqual(len(node_offers.get_accepted_new_job_exes()), 2)
self.assertSetEqual(set(node_offers.get_accepted_new_job_exes()),
{job_exe_1, job_exe_2})
self.assertListEqual(node_offers.get_accepted_running_job_exes(), [])
self.assertEqual(node_offers._available_cpus, 64.0)
self.assertEqual(node_offers._available_mem, 1536.0)
self.assertEqual(node_offers._available_disk, 2327.0)
def test_accept_new_job_exe_gpu_partial_node_other_task(self):
"""Tests successfully calling accept_new_job_exe() when job requires less GPUs than available"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(10.0), Mem(50.0), Gpus(1.0)])
task_resources = NodeResources([Gpus(1.0)])
watermark_resources = NodeResources(
[Cpus(100.0), Mem(500.0), Gpus(1.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
queue_model = queue_test_utils.create_queue(cpus_required=1.0,
mem_required=10.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0,
gpus_required=2)
job_exe = QueuedJobExecution(queue_model)
accepted = scheduling_node.accept_new_job_exe(job_exe)
self.assertFalse(accepted)
def test_lost_node(self):
"""Tests when the node is lost"""
node_offers = NodeOffers(self.node)
offer_1 = ResourceOffer('offer_1', self.node_agent, NodeResources(cpus=24.0, mem=1024.0, disk=1024.0))
node_offers.add_offer(offer_1)
offer_2 = ResourceOffer('offer_2', self.node_agent, NodeResources(cpus=50.0, mem=2048.0, disk=2048.0))
node_offers.add_offer(offer_2)
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_running_job_exes(), [])
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
# Accept a couple job executions
job_exe_1 = RunningJobExecution(self.running_job_exe_1)
result = node_offers.consider_next_task(job_exe_1)
self.assertEqual(result, NodeOffers.ACCEPTED)
job_exe_2 = QueuedJobExecution(self.queue_1)
result = node_offers.consider_new_job_exe(job_exe_2)
self.assertEqual(result, NodeOffers.ACCEPTED)
self.assertTrue(node_offers.has_accepted_job_exes())
self.assertGreater(node_offers._available_cpus, 0.0)
self.assertGreater(node_offers._available_mem, 0.0)
self.assertGreater(node_offers._available_disk, 0.0)
# Node is lost
node_offers.lost_node()
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertEqual(node_offers._available_cpus, 0.0)
self.assertEqual(node_offers._available_mem, 0.0)
self.assertEqual(node_offers._available_disk, 0.0)
def test_remove_offer(self):
"""Tests remove_offer()"""
node_offers = NodeOffers(self.node)
offer_1 = ResourceOffer(
'offer_1', self.node_agent,
NodeResources(cpus=24.0, mem=1024.0, disk=1024.0))
node_offers.add_offer(offer_1)
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=50.0, mem=2048.0, disk=2048.0))
node_offers.add_offer(offer_2)
job_exe_1 = QueuedJobExecution(self.queue_1)
result = node_offers.consider_new_job_exe(job_exe_1)
self.assertEqual(result, NodeOffers.ACCEPTED)
# Remove one offer, new job execution should still be accepted
node_offers.remove_offer(offer_1.id)
self.assertTrue(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_new_job_exes(),
[job_exe_1])
# Remove second offer, no resources left, all job executions should be removed
node_offers.remove_offer(offer_2.id)
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
self.assertEqual(node_offers._available_cpus, 0.0)
self.assertEqual(node_offers._available_mem, 0.0)
self.assertEqual(node_offers._available_disk, 0.0)
def test_score_job_exe_for_scheduling_insufficient_resources(self):
"""Tests calling score_job_exe_for_scheduling() when there are not enough resources to schedule the job"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(20.0), Mem(100.0)])
task_resources = NodeResources([Cpus(100.0), Mem(500.0)])
watermark_resources = NodeResources([Cpus(200.0), Mem(700.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
# Allocate 10 CPUs and 50 MiB memory to existing job execution
job_exe = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(10.0), Mem(50.0)]))
scheduling_node.accept_job_exe_next_task(job_exe, [])
# Should have 10 CPUs and 50 MiB memory left, so this job execution is too big
queue_model = queue_test_utils.create_queue(cpus_required=15.0,
mem_required=40.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe = QueuedJobExecution(queue_model)
score = scheduling_node.score_job_exe_for_scheduling(job_exe, [])
self.assertIsNone(score)
def _consider_new_job_exes(self):
"""Considers any queued job executions for scheduling
"""
if self._scheduler_manager.is_paused():
return
num_job_exes = 0
for queue in Queue.objects.get_queue():
job_type_id = queue.job_type_id
if job_type_id not in self._job_types or self._job_types[
job_type_id].is_paused:
continue
if job_type_id in self._job_type_limit_available and self._job_type_limit_available[
job_type_id] < 1:
continue
queued_job_exe = QueuedJobExecution(queue)
if self._offer_manager.consider_new_job_exe(
queued_job_exe) == OfferManager.ACCEPTED:
num_job_exes += 1
if job_type_id in self._job_type_limit_available:
self._job_type_limit_available[job_type_id] -= 1
if num_job_exes >= SchedulingThread.MAX_NEW_JOB_EXES:
break
def test_lost_node(self):
"""Tests accepting a running and queued job execution and then the node being lost"""
offer_1 = ResourceOffer(
'offer_1', self.node_agent,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
manager = OfferManager()
manager.add_new_offers([offer_1, offer_2])
manager.update_nodes([self.node, self.paused_node])
manager.ready_new_offers()
job_exe_1 = QueuedJobExecution(self.queue_1)
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.ACCEPTED)
job_exe_2 = RunningJobExecution(self.running_job_exe_2)
result = manager.consider_next_task(job_exe_2)
self.assertEqual(result, OfferManager.ACCEPTED)
manager.lost_node(self.node_agent)
node_offers = manager.pop_offers_with_accepted_job_exes()
self.assertEqual(len(node_offers), 0)
def test_accept_new_job_exe_no_jobs(self):
"""Tests calling accept_new_job_exe() when new job exes are not allowed"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = False
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(10.0), Mem(50.0)])
task_resources = NodeResources()
watermark_resources = NodeResources([Cpus(100.0), Mem(500.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
queue_model = queue_test_utils.create_queue(cpus_required=1.0,
mem_required=10.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe = QueuedJobExecution(queue_model)
accepted = scheduling_node.accept_new_job_exe(job_exe)
self.assertFalse(accepted)
self.assertEqual(len(scheduling_node._allocated_queued_job_exes), 0)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(NodeResources()))
self.assertTrue(
scheduling_node._remaining_resources.is_equal(
NodeResources([Cpus(10.0), Mem(50.0)])))
self.assertIsNone(job_exe._scheduled_node_id)
def test_paused_node(self):
"""Tests adding job executions when the node is paused"""
node_offers = NodeOffers(self.paused_node)
offer_1 = ResourceOffer('offer_1', self.node_agent_paused, NodeResources(cpus=24.0, mem=1024.0, disk=1024.0))
node_offers.add_offer(offer_1)
offer_2 = ResourceOffer('offer_2', self.node_agent_paused, NodeResources(cpus=50.0, mem=2048.0, disk=2048.0))
node_offers.add_offer(offer_2)
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_running_job_exes(), [])
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
# Ensure it accepts new tasks for already running job executions
job_exe_1 = RunningJobExecution(self.running_job_exe_1)
result = node_offers.consider_next_task(job_exe_1)
self.assertEqual(result, NodeOffers.ACCEPTED)
job_exe_2 = RunningJobExecution(self.running_job_exe_2)
result = node_offers.consider_next_task(job_exe_2)
self.assertEqual(result, NodeOffers.ACCEPTED)
# Don't accept new job executions while paused
job_exe_new = QueuedJobExecution(self.queue_1)
result = node_offers.consider_new_job_exe(job_exe_new)
self.assertEqual(result, NodeOffers.NODE_NOT_READY)
self.assertTrue(node_offers.has_accepted_job_exes())
self.assertEqual(len(node_offers.get_accepted_running_job_exes()), 2)
self.assertSetEqual(set(node_offers.get_accepted_running_job_exes()), {job_exe_1, job_exe_2})
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
self.assertEqual(node_offers._available_cpus, 68.0)
self.assertEqual(node_offers._available_mem, 1536.0)
self.assertEqual(node_offers._available_disk, 2222.0)
def test_reset_new_job_exes(self):
"""Tests calling reset_new_job_exes() successfully"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(100.0), Mem(500.0)])
watermark_resources = NodeResources([Cpus(100.0), Mem(500.0)])
resource_set = ResourceSet(offered_resources, NodeResources(),
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
queue_model_1 = queue_test_utils.create_queue(cpus_required=2.0,
mem_required=60.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe_1 = QueuedJobExecution(queue_model_1)
queue_model_2 = queue_test_utils.create_queue(cpus_required=4.5,
mem_required=400.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe_2 = QueuedJobExecution(queue_model_2)
allocated_resources = NodeResources()
allocated_resources.add(job_exe_1.required_resources)
allocated_resources.add(job_exe_2.required_resources)
# Set up node with queued job exes
scheduling_node.accept_new_job_exe(job_exe_1)
scheduling_node.accept_new_job_exe(job_exe_2)
self.assertEqual(len(scheduling_node._allocated_queued_job_exes), 2)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(allocated_resources))
# Reset queued job exes and check that everything is back to square one
scheduling_node.reset_new_job_exes()
self.assertEqual(len(scheduling_node._allocated_queued_job_exes), 0)
self.assertTrue(
scheduling_node.allocated_resources.is_equal(NodeResources()))
self.assertTrue(
scheduling_node._remaining_resources.is_equal(offered_resources))
def test_score_job_exe_for_scheduling(self):
"""Tests calling score_job_exe_for_scheduling() successfully"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(20.0), Mem(100.0)])
task_resources = NodeResources([Cpus(100.0), Mem(500.0)])
watermark_resources = NodeResources([Cpus(200.0), Mem(700.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
# Allocate 10 CPUs and 50 MiB memory to existing job execution
job_exe = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(10.0), Mem(50.0)]))
scheduling_node.accept_job_exe_next_task(job_exe, [])
# Should have 10 CPUs and 50 MiB memory left, so this should be scheduled
queue_model = queue_test_utils.create_queue(cpus_required=5.0,
mem_required=40.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe = QueuedJobExecution(queue_model)
# Expected available 85 CPUs and 110 MiB memory "left" on node
# (watermark - current tasks - allocated - new job we are scoring)
# First 2 job types should fit, next 2 are too big, so score should be 2
job_type_resource_1 = NodeResources([Cpus(2.0), Mem(10.0)])
job_type_resource_2 = NodeResources([Cpus(85.0), Mem(109.0)])
job_type_resource_3 = NodeResources([Cpus(86.0), Mem(10.0)])
job_type_resource_4 = NodeResources([Cpus(2.0), Mem(111.0)])
score = scheduling_node.score_job_exe_for_scheduling(
job_exe, [
job_type_resource_1, job_type_resource_2, job_type_resource_3,
job_type_resource_4
])
self.assertEqual(score, 2)
def test_no_offers(self):
"""Tests adding job executions when there are no offers"""
node_offers = NodeOffers(self.node)
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_running_job_exes(), [])
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
job_exe_1 = RunningJobExecution(self.running_job_exe_1)
result = node_offers.consider_next_task(job_exe_1)
self.assertEqual(result, NodeOffers.NO_OFFERS)
job_exe_new = QueuedJobExecution(self.queue_1)
result = node_offers.consider_new_job_exe(job_exe_new)
self.assertEqual(result, NodeOffers.NO_OFFERS)
self.assertFalse(node_offers.has_accepted_job_exes())
self.assertListEqual(node_offers.get_accepted_running_job_exes(), [])
self.assertListEqual(node_offers.get_accepted_new_job_exes(), [])
def test_successful(self):
"""Tests calling QueueManager.schedule_job_executions() successfully."""
queued_job_exe_1 = QueuedJobExecution(self.queue_1)
queued_job_exe_1.accepted(
self.node.id,
JobResources(cpus=self.queue_1.cpus_required,
mem=self.queue_1.mem_required,
disk_in=self.queue_1.disk_in_required,
disk_out=self.queue_1.disk_out_required,
disk_total=self.queue_1.disk_total_required))
queued_job_exe_2 = QueuedJobExecution(self.queue_2)
queued_job_exe_2.accepted(
self.node.id,
JobResources(cpus=self.queue_2.cpus_required,
mem=self.queue_2.mem_required,
disk_in=self.queue_2.disk_in_required,
disk_out=self.queue_2.disk_out_required,
disk_total=self.queue_2.disk_total_required))
scheduled_job_exes = Queue.objects.schedule_job_executions(
'framework-123', [queued_job_exe_1, queued_job_exe_2], {})
# Only job_exe_1 should have scheduled since job_exe_2 has an invalid job type configuration JSON
self.assertEqual(len(scheduled_job_exes), 1)
self.assertEqual(scheduled_job_exes[0].id, self.job_exe_1.id)
# job_exe_1 should be RUNNING
self.assertEqual(
Queue.objects.filter(job_exe_id=self.job_exe_1.id).count(), 0)
self.assertEqual(
JobExecution.objects.get(id=self.job_exe_1.id).status, 'RUNNING')
self.assertEqual(
Job.objects.get(id=self.job_exe_1.job_id).status, 'RUNNING')
# job_exe_2 should be QUEUED
self.assertEqual(
Queue.objects.filter(job_exe_id=self.job_exe_2.id).count(), 1)
self.assertEqual(
JobExecution.objects.get(id=self.job_exe_2.id).status, 'QUEUED')
self.assertEqual(
Job.objects.get(id=self.job_exe_2.job_id).status, 'QUEUED')
def test_no_ready_offers(self):
"""Tests considering job executions when no offers are ready"""
offer_1 = ResourceOffer(
'offer_1', self.node_agent_paused,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
manager = OfferManager()
manager.add_new_offers([offer_1, offer_2])
job_exe_1 = QueuedJobExecution(self.queue_1)
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.NO_NODES_AVAILABLE)
job_exe_2 = RunningJobExecution(self.running_job_exe_1)
result = manager.consider_next_task(job_exe_2)
self.assertEqual(result, OfferManager.NODE_OFFLINE)
def test_high_disk(self):
"""Tests rejecting a queued job execution due to too much disk required"""
offer_1 = ResourceOffer(
'offer_1', self.node_agent_paused,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
manager = OfferManager()
manager.add_new_offers([offer_1, offer_2])
manager.update_nodes([self.node, self.paused_node])
manager.ready_new_offers()
job_exe_1 = QueuedJobExecution(self.queue_high_disk)
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.NOT_ENOUGH_DISK)
node_offers = manager.pop_offers_with_accepted_job_exes()
self.assertEqual(len(node_offers), 0)
def test_all_offers_paused(self):
"""Tests rejecting a queued job execution due to all nodes being paused"""
offer_1 = ResourceOffer(
'offer_1', self.node_agent_paused,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent_paused,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
manager = OfferManager()
manager.add_new_offers([offer_1, offer_2])
manager.update_nodes([self.paused_node])
manager.ready_new_offers()
job_exe_1 = QueuedJobExecution(self.queue_1)
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.NO_NODES_AVAILABLE)
node_offers = manager.pop_offers_with_accepted_job_exes()
self.assertEqual(len(node_offers), 0)
def test_offers_with_no_nodes(self):
"""Tests considering job executions when offers cannot be readied due to no nodes updated"""
offer_1 = ResourceOffer(
'offer_1', self.node_agent_paused,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
manager = OfferManager()
manager.add_new_offers([offer_1, offer_2])
manager.ready_new_offers()
job_exe_1 = QueuedJobExecution(self.queue_1)
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.NO_NODES_AVAILABLE)
job_exe_2 = RunningJobExecution(self.running_job_exe_1)
result = manager.consider_next_task(job_exe_2)
self.assertEqual(result, OfferManager.NODE_NOT_READY)
def test_lost_node_that_comes_back(self):
"""Tests that when a lost name comes back, it can schedule tasks again"""
offer_1 = ResourceOffer(
'offer_1', self.node_agent,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
manager = OfferManager()
manager.add_new_offers([offer_1, offer_2])
manager.update_nodes([self.node])
manager.ready_new_offers()
# Node goes down and comes back up with new agent ID
manager.lost_node(self.node_agent)
new_node_agent = 'i_am_a_new_node_agent'
self.node.update_from_mesos(agent_id=new_node_agent)
job_exe_1 = QueuedJobExecution(self.queue_1)
# Offers for previous agent should be gone, do not schedule the job exe
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.NO_NODES_AVAILABLE)
offer_3 = ResourceOffer(
'offer_3', new_node_agent,
NodeResources(cpus=35.0, mem=3048.0, disk=3048.0))
manager.add_new_offers([offer_3])
manager.update_nodes([self.node])
manager.ready_new_offers()
# New offers have come in for new agent ID, should schedule job exe now
result = manager.consider_new_job_exe(job_exe_1)
self.assertEqual(result, OfferManager.ACCEPTED)
node_offers = manager.pop_offers_with_accepted_job_exes()
self.assertEqual(len(node_offers), 1)
def test_accept_new_job_exe_gpu_partial_node(self):
"""Tests successfully calling accept_new_job_exe() when job requires less GPUs than available"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(10.0), Mem(50.0), Gpus(4.0)])
task_resources = NodeResources()
watermark_resources = NodeResources(
[Cpus(100.0), Mem(500.0), Gpus(4.0)])
resource_set = ResourceSet(offered_resources, task_resources,
watermark_resources)
scheduling_node = SchedulingNode('agent_1', node, [], [], resource_set)
queue_model = queue_test_utils.create_queue(cpus_required=1.0,
mem_required=10.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0,
gpus_required=1)
job_exe = QueuedJobExecution(queue_model)
accepted = scheduling_node.accept_new_job_exe(job_exe)
self.assertTrue(accepted)
self.assertEqual(len(scheduling_node._allocated_queued_job_exes), 1)
# Verify that our greedy GPU allocation logic is working
self.assertTrue(
scheduling_node.allocated_resources.is_equal(
NodeResources([Cpus(1.0), Mem(10.0),
Gpus(4.0)])))
self.assertTrue(
scheduling_node._remaining_resources.is_equal(
NodeResources([Cpus(9.0), Mem(40.0)])))
self.assertEqual(job_exe._scheduled_node_id, node.id)
def _process_queue(self, nodes, job_types, job_type_limits,
job_type_resources, workspaces):
"""Retrieves the top of the queue and schedules new job executions on available nodes as resources and limits
allow
:param nodes: The dict of scheduling nodes stored by node ID for all nodes ready to accept new job executions
:type nodes: dict
:param job_types: The dict of job type models stored by job type ID
:type job_types: dict
:param job_type_limits: The dict of job type IDs mapping to job type limits
:type job_type_limits: dict
:param job_type_resources: The list of all of the job type resource requirements
:type job_type_resources: list
:param workspaces: A dict of all workspaces stored by name
:type workspaces: dict
:returns: The list of queued job executions that were scheduled
:rtype: list
"""
scheduled_job_executions = []
ignore_job_type_ids = self._calculate_job_types_to_ignore(
job_types, job_type_limits)
started = now()
for queue in Queue.objects.get_queue(
scheduler_mgr.config.queue_mode,
ignore_job_type_ids)[:QUEUE_LIMIT]:
job_exe = QueuedJobExecution(queue)
# Canceled job executions get processed as scheduled executions
if job_exe.is_canceled:
scheduled_job_executions.append(job_exe)
continue
# If there are no longer any available nodes, break
if not nodes:
break
# Make sure execution's job type and workspaces have been synced to the scheduler
job_type_id = queue.job_type_id
if job_type_id not in job_types:
continue
workspace_names = job_exe.configuration.get_input_workspace_names()
workspace_names.extend(
job_exe.configuration.get_output_workspace_names())
missing_workspace = False
for name in workspace_names:
missing_workspace = missing_workspace or name not in workspaces
if missing_workspace:
continue
# Check limit for this execution's job type
if job_type_id in job_type_limits and job_type_limits[
job_type_id] < 1:
continue
# Try to schedule job execution and adjust job type limit if needed
if self._schedule_new_job_exe(job_exe, nodes, job_type_resources):
scheduled_job_executions.append(job_exe)
if job_type_id in job_type_limits:
job_type_limits[job_type_id] -= 1
duration = now() - started
msg = 'Processing queue took %.3f seconds'
if duration > PROCESS_QUEUE_WARN_THRESHOLD:
logger.warning(msg, duration.total_seconds())
else:
logger.debug(msg, duration.total_seconds())
return scheduled_job_executions
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
node = node_test_utils.create_node()
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
recipe = Recipe.objects.get(id=recipe_id)
recipe_job_1 = RecipeJob.objects.select_related('job__job_exe').get(recipe_id=recipe_id, job_name='Job 1')
job_exe_1 = JobExecution.objects.get(job_id=recipe_job_1.job_id)
queued_job_exe = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_1.id))
queued_job_exe.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe], {})
results = JobResults()
results.add_file_list_parameter('Test Output 1', [product_test_utils.create_product().file_id])
JobExecution.objects.filter(id=job_exe_1.id).update(results=results.get_dict())
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(name=self.job_type_2.name, version='New Version',
interface=self.job_type_2.interface)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name': 'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_recipe_id = Queue.objects.queue_new_recipe(new_recipe_type, None, event, recipe, delta, superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=recipe_id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_recipe_id)
self.assertEqual(new_recipe.superseded_recipe_id, recipe_id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
# Complete both the old and new job 2 and check that only the new recipe completes
job_exe_2 = JobExecution.objects.get(job_id=recipe_job_2.job_id)
queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_2.id))
queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe_2], {})
Queue.objects.handle_job_completion(job_exe_2.id, now())
new_job_exe_2 = JobExecution.objects.get(job_id=new_recipe_job_2.job_id)
new_queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=new_job_exe_2.id))
new_queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000,
disk_total=2000))
Queue.objects.schedule_job_executions('123', [new_queued_job_exe_2], {})
Queue.objects.handle_job_completion(new_job_exe_2.id, now())
recipe = Recipe.objects.get(id=recipe.id)
new_recipe = Recipe.objects.get(id=new_recipe.id)
self.assertIsNone(recipe.completed)
self.assertIsNotNone(new_recipe.completed)
def _process_queue(self, nodes, job_types, job_type_limits,
job_type_resources, workspaces):
"""Retrieves the top of the queue and schedules new job executions on available nodes as resources and limits
allow
:param nodes: The dict of scheduling nodes stored by node ID for all nodes ready to accept new job executions
:type nodes: dict
:param job_types: The dict of job type models stored by job type ID
:type job_types: dict
:param job_type_limits: The dict of job type IDs mapping to job type limits
:type job_type_limits: dict
:param job_type_resources: The list of all of the job type resource requirements
:type job_type_resources: list
:param workspaces: A dict of all workspaces stored by name
:type workspaces: dict
:returns: The list of queued job executions that were scheduled
:rtype: list
"""
scheduled_job_executions = []
started = now()
type_warnings = {}
# We can schedule as long as there are nodes
if not nodes:
logger.warning(
'There are no nodes available. Waiting to schedule until there are free resources...'
)
return scheduled_job_executions
ignore_job_type_ids = self._calculate_job_types_to_ignore(
job_types, job_type_limits)
max_cluster_resources = resource_mgr.get_max_available_resources()
for queue in Queue.objects.get_queue(
scheduler_mgr.config.queue_mode,
ignore_job_type_ids)[:QUEUE_LIMIT]:
job_exe = QueuedJobExecution(queue)
# Canceled job executions get processed as scheduled executions
if job_exe.is_canceled:
scheduled_job_executions.append(job_exe)
continue
jt = job_type_mgr.get_job_type(queue.job_type.id)
name = INVALID_RESOURCES.name + jt.name
title = INVALID_RESOURCES.title % jt.name
warning = SchedulerWarning(name=name,
title=title,
description=None)
if jt.unmet_resources and scheduler_mgr.is_warning_active(warning):
# previously checked this job type and found we lacked resources; wait until warning is inactive to check again
continue
invalid_resources = []
insufficient_resources = []
# get resource names offered and compare to job type resources
for resource in job_exe.required_resources.resources:
# Check for invalid resource or sharedmem
if (resource.name not in max_cluster_resources._resources) or (
resource.name.lower() == 'sharedmem'):
# Skip sharedmem if its 0
if (resource.name.lower()
== 'sharedmem') and (resource.value <= 0):
continue
if jt.name in type_warnings:
type_warnings[jt.name]['count'] += 1
if resource.name not in type_warnings[
jt.name]['warning']:
type_warnings[jt.name]['warning'] += (
', %s' % resource.name)
else:
type_warnings[jt.name] = {
'warning':
'%s job types could not be scheduled as the following resources do not exist in the available cluster resources: %s'
% (jt.name, resource.name),
'count':
1
}
# resource does not exist in cluster
invalid_resources.append(resource.name)
elif resource.value > max_cluster_resources._resources[
resource.name].value:
# resource exceeds the max available from any node
insufficient_resources.append(resource.name)
if invalid_resources:
description = INVALID_RESOURCES.description % invalid_resources
scheduler_mgr.warning_active(warning, description)
if insufficient_resources:
description = INSUFFICIENT_RESOURCES.description % insufficient_resources
scheduler_mgr.warning_active(warning, description)
if invalid_resources or insufficient_resources:
invalid_resources.extend(insufficient_resources)
jt.unmet_resources = ','.join(invalid_resources)
jt.save(update_fields=["unmet_resources"])
continue
else:
# reset unmet_resources flag
jt.unmet_resources = None
scheduler_mgr.warning_inactive(warning)
jt.save(update_fields=["unmet_resources"])
# Make sure execution's job type and workspaces have been synced to the scheduler
job_type_id = queue.job_type_id
if job_type_id not in job_types:
scheduler_mgr.warning_active(
UNKNOWN_JOB_TYPE,
description=UNKNOWN_JOB_TYPE.description % job_type_id)
continue
workspace_names = job_exe.configuration.get_input_workspace_names()
workspace_names.extend(
job_exe.configuration.get_output_workspace_names())
missing_workspace = False
for name in workspace_names:
missing_workspace = missing_workspace or name not in workspaces
if missing_workspace:
if jt.name in type_warnings:
type_warnings[jt.name]['count'] += 1
else:
type_warnings[jt.name] = {
'warning':
'%s job types could not be scheduled due to missing workspace'
% jt.name,
'count':
1
}
continue
# Check limit for this execution's job type
if job_type_id in job_type_limits and job_type_limits[
job_type_id] < 1:
if jt.name in type_warnings:
type_warnings[jt.name]['count'] += 1
else:
type_warnings[jt.name] = {
'warning':
'%s job types could not be scheduled due to scheduling limit reached'
% jt.name,
'count':
1
}
continue
# Try to schedule job execution and adjust job type limit if needed
if self._schedule_new_job_exe(job_exe, nodes, job_type_resources):
scheduled_job_executions.append(job_exe)
if job_type_id in job_type_limits:
job_type_limits[job_type_id] -= 1
duration = now() - started
if type_warnings:
for warn in type_warnings:
logger.warning('%d %s', type_warnings[warn]['count'],
type_warnings[warn]['warning'])
msg = 'Processing queue took %.3f seconds'
if duration > PROCESS_QUEUE_WARN_THRESHOLD:
logger.warning(msg, duration.total_seconds())
else:
logger.debug(msg, duration.total_seconds())
return scheduled_job_executions
def test_score_job_exe_for_reservation(self):
"""Tests calling score_job_exe_for_reservation() successfully"""
node = MagicMock()
node.hostname = 'host_1'
node.id = 1
node.is_ready_for_new_job = MagicMock()
node.is_ready_for_new_job.return_value = True
node.is_ready_for_next_job_task = MagicMock()
node.is_ready_for_next_job_task.return_value = True
offered_resources = NodeResources([Cpus(20.0), Mem(100.0)])
watermark_resources = NodeResources([Cpus(200.0), Mem(700.0)])
resource_set = ResourceSet(offered_resources, NodeResources(),
watermark_resources)
task = HealthTask(
'1234', 'agent_1') # Resources are 0.1 CPUs and 32 MiB memory
job_exe_1 = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(10.0), Mem(50.0)]),
priority=1000)
job_exe_2 = job_test_utils.create_running_job_exe(
agent_id=self.agent_id,
resources=NodeResources([Cpus(56.0), Mem(15.0)]),
priority=100)
scheduling_node = SchedulingNode('agent_1', node, [task],
[job_exe_1, job_exe_2], resource_set)
queue_model_1 = queue_test_utils.create_queue(priority=100,
cpus_required=8.0,
mem_required=40.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe_1 = QueuedJobExecution(queue_model_1)
queue_model_2 = queue_test_utils.create_queue(priority=1000,
cpus_required=8.0,
mem_required=40.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe_2 = QueuedJobExecution(queue_model_2)
scheduling_node.accept_new_job_exe(job_exe_1)
scheduling_node.accept_new_job_exe(job_exe_2)
# We are going to try to reserve the node for a job execution with priority 120
# Calculate available resources for reservation:
# Watermark (200, 700) - System Tasks (0.1, 32) - Higher Priority Existing Job Exes (56, 15) - Higher Priority
# New Job Exes (8, 40) = 135.9 CPUs, 613 memory
# This new job should fit for reservation
queue_model = queue_test_utils.create_queue(priority=120,
cpus_required=130.0,
mem_required=600.0,
disk_in_required=0.0,
disk_out_required=0.0,
disk_total_required=0.0)
job_exe = QueuedJobExecution(queue_model)
# Expected available 5.9 CPUs and 13 MiB memory "left" on node
# (available above - new job we are scoring)
# First 2 job types should fit, next 2 are too big, so score should be 2
job_type_resource_1 = NodeResources([Cpus(2.0), Mem(10.0)])
job_type_resource_2 = NodeResources([Cpus(5.5), Mem(12.0)])
job_type_resource_3 = NodeResources([Cpus(6.0), Mem(10.0)])
job_type_resource_4 = NodeResources([Cpus(2.0), Mem(14.0)])
score = scheduling_node.score_job_exe_for_reservation(
job_exe, [
job_type_resource_1, job_type_resource_2, job_type_resource_3,
job_type_resource_4
])
self.assertEqual(score, 2)