def test_job_type_limit(self, mock_taskinfo):
"""Tests running the scheduling thread with a job type limit"""
mock_taskinfo.return_value = MagicMock()
Queue.objects.all().delete()
job_type_with_limit = job_test_utils.create_job_type()
job_type_with_limit.max_scheduled = 4
job_type_with_limit.save()
job_exe_1 = job_test_utils.create_job_exe(job_type=job_type_with_limit, status='RUNNING')
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
job_type_mgr.sync_with_database()
# One job of this type is already running
running_job_mgr.add_job_exes([RunningJobExecution(job_exe_1)])
offer_1 = ResourceOffer('offer_1', self.node_agent_1, NodeResources(cpus=200.0, mem=102400.0, disk=102400.0))
offer_2 = ResourceOffer('offer_2', self.node_agent_2, NodeResources(cpus=200.0, mem=204800.0, disk=204800.0))
offer_mgr.add_new_offers([offer_1, offer_2])
# Ignore cleanup tasks
for node in node_mgr.get_nodes():
node.initial_cleanup_completed()
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(num_tasks, 3) # One is already running, should only be able to schedule 3 more
def test_successful_schedule(self, mock_taskinfo):
"""Tests successfully scheduling tasks"""
mock_taskinfo.return_value = MagicMock()
offer_1 = ResourceOffer('offer_1', self.node_agent_1, NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer('offer_2', self.node_agent_2, NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
offer_mgr.add_new_offers([offer_1, offer_2])
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(num_tasks, 2) # Schedule both queued job executions
def test_paused_scheduler(self, mock_taskinfo):
"""Tests running the scheduling thread with a paused scheduler"""
mock_taskinfo.return_value = MagicMock()
offer_1 = ResourceOffer('offer_1', self.node_agent_1, NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer('offer_2', self.node_agent_2, NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
offer_mgr.add_new_offers([offer_1, offer_2])
Scheduler.objects.update(is_paused=True)
scheduler_mgr.sync_with_database()
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(num_tasks, 0)
def test_successful_schedule(self, mock_taskinfo):
"""Tests successfully scheduling tasks"""
mock_taskinfo.return_value = MagicMock()
offer_1 = ResourceOffer(
'offer_1', self.node_agent_1,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent_2,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
offer_mgr.add_new_offers([offer_1, offer_2])
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(num_tasks, 2) # Schedule both queued job executions
def resourceOffers(self, driver, offers):
"""
Invoked when resources have been offered to this framework. A single
offer will only contain resources from a single slave. Resources
associated with an offer will not be re-offered to _this_ framework
until either (a) this framework has rejected those resources (see
SchedulerDriver.launchTasks) or (b) those resources have been
rescinded (see Scheduler.offerRescinded). Note that resources may be
concurrently offered to more than one framework at a time (depending
on the allocator being used). In that case, the first framework to
launch tasks using those resources will be able to use them while the
other frameworks will have those resources rescinded (or if a
framework has already launched tasks with those resources then those
tasks will fail with a TASK_LOST status and a message saying as much).
See documentation for :meth:`mesos_api.mesos.Scheduler.resourceOffers`.
"""
started = now()
agent_ids = []
resource_offers = []
for offer in offers:
offer_id = offer.id.value
agent_id = offer.slave_id.value
disk = 0
mem = 0
cpus = 0
for resource in offer.resources:
if resource.name == 'disk':
disk = resource.scalar.value
elif resource.name == 'mem':
mem = resource.scalar.value
elif resource.name == 'cpus':
cpus = resource.scalar.value
resources = NodeResources(cpus=cpus, mem=mem, disk=disk)
agent_ids.append(agent_id)
resource_offers.append(ResourceOffer(offer_id, agent_id, resources))
node_mgr.register_agent_ids(agent_ids)
offer_mgr.add_new_offers(resource_offers)
duration = now() - started
msg = 'Scheduler resourceOffers() took %.3f seconds'
if duration > ScaleScheduler.NORMAL_WARN_THRESHOLD:
logger.warning(msg, duration.total_seconds())
else:
logger.debug(msg, duration.total_seconds())
def test_paused_scheduler(self, mock_taskinfo):
"""Tests running the scheduling thread with a paused scheduler"""
mock_taskinfo.return_value = MagicMock()
offer_1 = ResourceOffer(
'offer_1', self.node_agent_1,
NodeResources(cpus=2.0, mem=1024.0, disk=1024.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent_2,
NodeResources(cpus=25.0, mem=2048.0, disk=2048.0))
offer_mgr.add_new_offers([offer_1, offer_2])
Scheduler.objects.update(is_paused=True)
scheduler_mgr.sync_with_database()
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(num_tasks, 0)
def test_job_type_limit(self, mock_taskinfo):
"""Tests running the scheduling thread with a job type limit"""
mock_taskinfo.return_value = MagicMock()
Queue.objects.all().delete()
job_type_with_limit = job_test_utils.create_job_type()
job_type_with_limit.max_scheduled = 4
job_type_with_limit.save()
job_exe_1 = job_test_utils.create_job_exe(job_type=job_type_with_limit,
status='RUNNING')
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
job_type_mgr.sync_with_database()
# One job of this type is already running
job_exe_mgr.schedule_job_exes([RunningJobExecution(job_exe_1)])
offer_1 = ResourceOffer(
'offer_1', self.node_agent_1,
NodeResources(cpus=200.0, mem=102400.0, disk=102400.0))
offer_2 = ResourceOffer(
'offer_2', self.node_agent_2,
NodeResources(cpus=200.0, mem=204800.0, disk=204800.0))
offer_mgr.add_new_offers([offer_1, offer_2])
# Ignore Docker pull tasks
for node in node_mgr.get_nodes():
node._is_image_pulled = True
# Ignore cleanup tasks
for node in node_mgr.get_nodes():
node._initial_cleanup_completed()
node._update_state()
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(
num_tasks, 3
) # One is already running, should only be able to schedule 3 more
