def test_job_type_limit(self, mock_taskinfo): """Tests running the scheduling thread with a job type limit""" mock_taskinfo.return_value = MagicMock() Queue.objects.all().delete() job_type_with_limit = job_test_utils.create_job_type() job_type_with_limit.max_scheduled = 4 job_type_with_limit.save() job_exe_1 = job_test_utils.create_job_exe(job_type=job_type_with_limit, status='RUNNING') queue_test_utils.create_queue(job_type=job_type_with_limit) queue_test_utils.create_queue(job_type=job_type_with_limit) queue_test_utils.create_queue(job_type=job_type_with_limit) queue_test_utils.create_queue(job_type=job_type_with_limit) queue_test_utils.create_queue(job_type=job_type_with_limit) queue_test_utils.create_queue(job_type=job_type_with_limit) job_type_mgr.sync_with_database() # One job of this type is already running running_job_mgr.add_job_exes([RunningJobExecution(job_exe_1)]) offer_1 = ResourceOffer('offer_1', self.node_agent_1, NodeResources(cpus=200.0, mem=102400.0, disk=102400.0)) offer_2 = ResourceOffer('offer_2', self.node_agent_2, NodeResources(cpus=200.0, mem=204800.0, disk=204800.0)) offer_mgr.add_new_offers([offer_1, offer_2]) # Ignore cleanup tasks for node in node_mgr.get_nodes(): node.initial_cleanup_completed() num_tasks = self._scheduling_thread._perform_scheduling() self.assertEqual(num_tasks, 3) # One is already running, should only be able to schedule 3 more
def _schedule_accepted_tasks(self): """Schedules all of the tasks that have been accepted :returns: The number of Mesos tasks that were scheduled :rtype: int """ when = now() tasks_to_launch = {} # {Node ID: [Mesos Tasks]} queued_job_exes_to_schedule = [] node_offers_list = offer_mgr.pop_offers_with_accepted_job_exes() for node_offers in node_offers_list: mesos_tasks = [] tasks_to_launch[node_offers.node.id] = mesos_tasks # Add cleanup tasks for task in node_offers.get_accepted_tasks(): task.launch(when) mesos_tasks.append(create_mesos_task(task)) # Start next task for already running job executions that were accepted for running_job_exe in node_offers.get_accepted_running_job_exes(): task = running_job_exe.start_next_task() if task: task.launch(when) mesos_tasks.append(create_mesos_task(task)) # Gather up queued job executions that were accepted for queued_job_exe in node_offers.get_accepted_new_job_exes(): queued_job_exes_to_schedule.append(queued_job_exe) try: # Schedule queued job executions and start their first tasks workspaces = workspace_mgr.get_workspaces() scheduled_job_exes = self._schedule_queued_job_executions(queued_job_exes_to_schedule, workspaces) running_job_mgr.add_job_exes(scheduled_job_exes) for scheduled_job_exe in scheduled_job_exes: task = scheduled_job_exe.start_next_task() if task: task.launch(when) tasks_to_launch[scheduled_job_exe.node_id].append(create_mesos_task(task)) except OperationalError: logger.exception('Failed to schedule queued job executions') # Launch tasks on Mesos total_num_tasks = 0 total_num_nodes = 0 for node_offers in node_offers_list: task_list = tasks_to_launch[node_offers.node.id] num_tasks = len(task_list) total_num_tasks += num_tasks if num_tasks: total_num_nodes += 1 mesos_offer_ids = [] for offer_id in node_offers.offer_ids: mesos_offer_id = mesos_pb2.OfferID() mesos_offer_id.value = offer_id mesos_offer_ids.append(mesos_offer_id) self._driver.launchTasks(mesos_offer_ids, task_list) if total_num_tasks: logger.info('Launched %i Mesos task(s) on %i node(s)', total_num_tasks, total_num_nodes) return total_num_tasks