def test_job_type_limit(self, mock_taskinfo):
"""Tests running the scheduling thread with a job type limit"""
mock_taskinfo.return_value = MagicMock()
Queue.objects.all().delete()
job_type_with_limit = job_test_utils.create_job_type()
job_type_with_limit.max_scheduled = 4
job_type_with_limit.save()
job_exe_1 = job_test_utils.create_job_exe(job_type=job_type_with_limit, status='RUNNING')
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
queue_test_utils.create_queue(job_type=job_type_with_limit)
job_type_mgr.sync_with_database()
# One job of this type is already running
running_job_mgr.add_job_exes([RunningJobExecution(job_exe_1)])
offer_1 = ResourceOffer('offer_1', self.node_agent_1, NodeResources(cpus=200.0, mem=102400.0, disk=102400.0))
offer_2 = ResourceOffer('offer_2', self.node_agent_2, NodeResources(cpus=200.0, mem=204800.0, disk=204800.0))
offer_mgr.add_new_offers([offer_1, offer_2])
# Ignore cleanup tasks
for node in node_mgr.get_nodes():
node.initial_cleanup_completed()
num_tasks = self._scheduling_thread._perform_scheduling()
self.assertEqual(num_tasks, 3) # One is already running, should only be able to schedule 3 more
def _schedule_accepted_tasks(self):
"""Schedules all of the tasks that have been accepted
:returns: The number of Mesos tasks that were scheduled
:rtype: int
"""
when = now()
tasks_to_launch = {} # {Node ID: [Mesos Tasks]}
queued_job_exes_to_schedule = []
node_offers_list = offer_mgr.pop_offers_with_accepted_job_exes()
for node_offers in node_offers_list:
mesos_tasks = []
tasks_to_launch[node_offers.node.id] = mesos_tasks
# Add cleanup tasks
for task in node_offers.get_accepted_tasks():
task.launch(when)
mesos_tasks.append(create_mesos_task(task))
# Start next task for already running job executions that were accepted
for running_job_exe in node_offers.get_accepted_running_job_exes():
task = running_job_exe.start_next_task()
if task:
task.launch(when)
mesos_tasks.append(create_mesos_task(task))
# Gather up queued job executions that were accepted
for queued_job_exe in node_offers.get_accepted_new_job_exes():
queued_job_exes_to_schedule.append(queued_job_exe)
try:
# Schedule queued job executions and start their first tasks
workspaces = workspace_mgr.get_workspaces()
scheduled_job_exes = self._schedule_queued_job_executions(queued_job_exes_to_schedule, workspaces)
running_job_mgr.add_job_exes(scheduled_job_exes)
for scheduled_job_exe in scheduled_job_exes:
task = scheduled_job_exe.start_next_task()
if task:
task.launch(when)
tasks_to_launch[scheduled_job_exe.node_id].append(create_mesos_task(task))
except OperationalError:
logger.exception('Failed to schedule queued job executions')
# Launch tasks on Mesos
total_num_tasks = 0
total_num_nodes = 0
for node_offers in node_offers_list:
task_list = tasks_to_launch[node_offers.node.id]
num_tasks = len(task_list)
total_num_tasks += num_tasks
if num_tasks:
total_num_nodes += 1
mesos_offer_ids = []
for offer_id in node_offers.offer_ids:
mesos_offer_id = mesos_pb2.OfferID()
mesos_offer_id.value = offer_id
mesos_offer_ids.append(mesos_offer_id)
self._driver.launchTasks(mesos_offer_ids, task_list)
if total_num_tasks:
logger.info('Launched %i Mesos task(s) on %i node(s)', total_num_tasks, total_num_nodes)
return total_num_tasks