def _schedule_accepted_tasks(self): """Schedules all of the tasks that have been accepted :returns: The number of Mesos tasks that were scheduled :rtype: int """ when = now() tasks_to_launch = {} # {Node ID: [Mesos Tasks]} queued_job_exes_to_schedule = [] node_offers_list = offer_mgr.pop_offers_with_accepted_job_exes() for node_offers in node_offers_list: mesos_tasks = [] tasks_to_launch[node_offers.node.id] = mesos_tasks # Add cleanup tasks for task in node_offers.get_accepted_tasks(): task.launch(when) mesos_tasks.append(create_mesos_task(task)) # Start next task for already running job executions that were accepted for running_job_exe in node_offers.get_accepted_running_job_exes(): task = running_job_exe.start_next_task() if task: task.launch(when) mesos_tasks.append(create_mesos_task(task)) # Gather up queued job executions that were accepted for queued_job_exe in node_offers.get_accepted_new_job_exes(): queued_job_exes_to_schedule.append(queued_job_exe) try: # Schedule queued job executions and start their first tasks workspaces = workspace_mgr.get_workspaces() scheduled_job_exes = self._schedule_queued_job_executions(queued_job_exes_to_schedule, workspaces) running_job_mgr.add_job_exes(scheduled_job_exes) for scheduled_job_exe in scheduled_job_exes: task = scheduled_job_exe.start_next_task() if task: task.launch(when) tasks_to_launch[scheduled_job_exe.node_id].append(create_mesos_task(task)) except OperationalError: logger.exception('Failed to schedule queued job executions') # Launch tasks on Mesos total_num_tasks = 0 total_num_nodes = 0 for node_offers in node_offers_list: task_list = tasks_to_launch[node_offers.node.id] num_tasks = len(task_list) total_num_tasks += num_tasks if num_tasks: total_num_nodes += 1 mesos_offer_ids = [] for offer_id in node_offers.offer_ids: mesos_offer_id = mesos_pb2.OfferID() mesos_offer_id.value = offer_id mesos_offer_ids.append(mesos_offer_id) self._driver.launchTasks(mesos_offer_ids, task_list) if total_num_tasks: logger.info('Launched %i Mesos task(s) on %i node(s)', total_num_tasks, total_num_nodes) return total_num_tasks
def perform_scheduling(self, client, when): """Organizes and analyzes the cluster resources, schedules new job executions, and launches tasks :param client: The Mesos scheduler client :type client: :class:`mesoshttp.client.MesosClient` :param when: The current time :type when: :class:`datetime.datetime` :returns: The number of tasks that were scheduled :rtype: int """ # Get framework ID first to make sure it doesn't change throughout scheduling process framework_id = scheduler_mgr.framework_id if not framework_id or not client or not client.get_driver(): # Don't schedule anything until the scheduler has connected to Mesos logger.warning( 'Scheduler not connected to Mesos. Scheduling delayed until connection established.' ) return 0 resource_mgr.update_all_cluster_resources() job_types = job_type_mgr.get_job_types() job_type_resources = job_type_mgr.get_job_type_resources() tasks = task_mgr.get_all_tasks() running_job_exes = job_exe_mgr.get_running_job_exes() workspaces = workspace_mgr.get_workspaces() nodes = self._prepare_nodes(tasks, running_job_exes, when) fulfilled_nodes = self._schedule_waiting_tasks(nodes, running_job_exes, when) sys_tasks_scheduled = self._schedule_system_tasks( fulfilled_nodes, job_type_resources, when) job_exe_count = 0 if sys_tasks_scheduled: # Only schedule new job executions if all needed system tasks have been scheduled job_type_limits = self._calculate_job_type_limits( job_types, running_job_exes) job_exe_count = self._schedule_new_job_exes( framework_id, fulfilled_nodes, job_types, job_type_limits, job_type_resources, workspaces) else: logger.warning('No new jobs scheduled due to waiting system tasks') scheduler_mgr.warning_active(WAITING_SYSTEM_TASKS) if framework_id != scheduler_mgr.framework_id: logger.warning( 'Scheduler framework ID changed, skipping task launch') return 0 self._allocate_offers(nodes) declined = resource_mgr.decline_offers() self._decline_offers(declined) task_count, offer_count = self._launch_tasks(client, nodes) scheduler_mgr.add_scheduling_counts(job_exe_count, task_count, offer_count) return task_count
def perform_scheduling(self, driver, when): """Organizes and analyzes the cluster resources, schedules new job executions, and launches tasks :param driver: The Mesos scheduler driver :type driver: :class:`mesos_api.mesos.SchedulerDriver` :param when: The current time :type when: :class:`datetime.datetime` :returns: The number of tasks that were scheduled :rtype: int """ # Get framework ID first to make sure it doesn't change throughout scheduling process framework_id = scheduler_mgr.framework_id job_types = job_type_mgr.get_job_types() job_type_resources = job_type_mgr.get_job_type_resources() tasks = task_mgr.get_all_tasks() running_job_exes = job_exe_mgr.get_running_job_exes() workspaces = workspace_mgr.get_workspaces() nodes = self._prepare_nodes(tasks, running_job_exes, when) fulfilled_nodes = self._schedule_waiting_tasks(nodes, running_job_exes, when) sys_tasks_scheduled = self._schedule_system_tasks( fulfilled_nodes, job_type_resources, when) job_exe_count = 0 if sys_tasks_scheduled: # Only schedule new job executions if all needed system tasks have been scheduled job_type_limits = self._calculate_job_type_limits( job_types, running_job_exes) job_exe_count = self._schedule_new_job_exes( framework_id, fulfilled_nodes, job_types, job_type_limits, job_type_resources, workspaces) else: # TODO: this is a good place for a scheduler warning in the status JSON logger.warning('No new jobs scheduled due to waiting system tasks') if framework_id != scheduler_mgr.framework_id: logger.warning( 'Scheduler framework ID changed, skipping task launch') return 0 self._allocate_offers(nodes) task_count, offer_count = self._launch_tasks(driver, nodes) scheduler_mgr.add_scheduling_counts(job_exe_count, task_count, offer_count) return task_count
def _schedule_accepted_tasks(self): """Schedules all of the tasks that have been accepted :returns: The number of Mesos tasks that were scheduled :rtype: int """ when = now() tasks = [] tasks_to_launch = {} # {Node ID: [Mesos Tasks]} queued_job_exes_to_schedule = [] node_offers_list = offer_mgr.pop_offers_with_accepted_job_exes() for node_offers in node_offers_list: mesos_tasks = [] tasks_to_launch[node_offers.node.id] = mesos_tasks # Add cleanup tasks for task in node_offers.get_accepted_tasks(): tasks.append(task) mesos_tasks.append(create_mesos_task(task)) # Start next task for already running job executions that were accepted for running_job_exe in node_offers.get_accepted_running_job_exes(): task = running_job_exe.start_next_task() if task: tasks.append(task) mesos_tasks.append(create_mesos_task(task)) # Gather up queued job executions that were accepted for queued_job_exe in node_offers.get_accepted_new_job_exes(): queued_job_exes_to_schedule.append(queued_job_exe) try: # Schedule queued job executions and start their first tasks workspaces = workspace_mgr.get_workspaces() scheduled_job_exes = self._schedule_queued_job_executions(queued_job_exes_to_schedule, workspaces) job_exe_mgr.schedule_job_exes(scheduled_job_exes) for scheduled_job_exe in scheduled_job_exes: task = scheduled_job_exe.start_next_task() if task: tasks.append(task) tasks_to_launch[scheduled_job_exe.node_id].append(create_mesos_task(task)) except OperationalError: logger.exception('Failed to schedule queued job executions') # Launch tasks on Mesos task_mgr.launch_tasks(tasks, when) total_num_tasks = 0 total_num_nodes = 0 for node_offers in node_offers_list: task_list = tasks_to_launch[node_offers.node.id] num_tasks = len(task_list) total_num_tasks += num_tasks if num_tasks: total_num_nodes += 1 mesos_offer_ids = [] for offer_id in node_offers.offer_ids: mesos_offer_id = mesos_pb2.OfferID() mesos_offer_id.value = offer_id mesos_offer_ids.append(mesos_offer_id) self._driver.launchTasks(mesos_offer_ids, task_list) if total_num_tasks: logger.info('Launched %i Mesos task(s) on %i node(s)', total_num_tasks, total_num_nodes) return total_num_tasks