def _perform_scheduling(self):
"""Performs task reconciliation with the Mesos master
:returns: The number of Mesos tasks that were scheduled
:rtype: int
"""
when = now()
# Get updated node and job type models from managers
nodes = node_mgr.get_nodes()
cleanup_mgr.update_nodes(nodes)
offer_mgr.update_nodes(nodes)
offer_mgr.ready_new_offers()
self._job_types = job_type_mgr.get_job_types()
# Look at job type limits and determine number available to be scheduled
self._job_type_limit_available = {}
for job_type in self._job_types.values():
if job_type.max_scheduled:
self._job_type_limit_available[job_type.id] = job_type.max_scheduled
for running_job_exe in job_exe_mgr.get_running_job_exes():
if running_job_exe.job_type_id in self._job_type_limit_available:
self._job_type_limit_available[running_job_exe.job_type_id] -= 1
self._consider_node_tasks(when)
self._consider_running_job_exes()
self._consider_new_job_exes()
return self._schedule_accepted_tasks()
def perform_scheduling(self, client, when):
"""Organizes and analyzes the cluster resources, schedules new job executions, and launches tasks
:param client: The Mesos scheduler client
:type client: :class:`mesoshttp.client.MesosClient`
:param when: The current time
:type when: :class:`datetime.datetime`
:returns: The number of tasks that were scheduled
:rtype: int
"""
# Get framework ID first to make sure it doesn't change throughout scheduling process
framework_id = scheduler_mgr.framework_id
if not framework_id or not client or not client.get_driver():
# Don't schedule anything until the scheduler has connected to Mesos
logger.warning(
'Scheduler not connected to Mesos. Scheduling delayed until connection established.'
)
return 0
resource_mgr.update_all_cluster_resources()
job_types = job_type_mgr.get_job_types()
job_type_resources = job_type_mgr.get_job_type_resources()
tasks = task_mgr.get_all_tasks()
running_job_exes = job_exe_mgr.get_running_job_exes()
workspaces = workspace_mgr.get_workspaces()
nodes = self._prepare_nodes(tasks, running_job_exes, when)
fulfilled_nodes = self._schedule_waiting_tasks(nodes, running_job_exes,
when)
sys_tasks_scheduled = self._schedule_system_tasks(
fulfilled_nodes, job_type_resources, when)
job_exe_count = 0
if sys_tasks_scheduled:
# Only schedule new job executions if all needed system tasks have been scheduled
job_type_limits = self._calculate_job_type_limits(
job_types, running_job_exes)
job_exe_count = self._schedule_new_job_exes(
framework_id, fulfilled_nodes, job_types, job_type_limits,
job_type_resources, workspaces)
else:
logger.warning('No new jobs scheduled due to waiting system tasks')
scheduler_mgr.warning_active(WAITING_SYSTEM_TASKS)
if framework_id != scheduler_mgr.framework_id:
logger.warning(
'Scheduler framework ID changed, skipping task launch')
return 0
self._allocate_offers(nodes)
declined = resource_mgr.decline_offers()
self._decline_offers(declined)
task_count, offer_count = self._launch_tasks(client, nodes)
scheduler_mgr.add_scheduling_counts(job_exe_count, task_count,
offer_count)
return task_count
def _reconcile_running_jobs(self):
"""Reconciles all currently running job executions with Mesos"""
# List of tasks to reconcile
tasks_to_reconcile = []
# Find current tasks for running executions
for running_job_exe in job_exe_mgr.get_running_job_exes():
task = running_job_exe.current_task
if task:
tasks_to_reconcile.append(task)
# Send tasks to reconciliation thread
recon_mgr.add_tasks(tasks_to_reconcile)
def perform_scheduling(self, driver, when):
"""Organizes and analyzes the cluster resources, schedules new job executions, and launches tasks
:param driver: The Mesos scheduler driver
:type driver: :class:`mesos_api.mesos.SchedulerDriver`
:param when: The current time
:type when: :class:`datetime.datetime`
:returns: The number of tasks that were scheduled
:rtype: int
"""
# Get framework ID first to make sure it doesn't change throughout scheduling process
framework_id = scheduler_mgr.framework_id
job_types = job_type_mgr.get_job_types()
job_type_resources = job_type_mgr.get_job_type_resources()
tasks = task_mgr.get_all_tasks()
running_job_exes = job_exe_mgr.get_running_job_exes()
workspaces = workspace_mgr.get_workspaces()
nodes = self._prepare_nodes(tasks, running_job_exes, when)
fulfilled_nodes = self._schedule_waiting_tasks(nodes, running_job_exes,
when)
sys_tasks_scheduled = self._schedule_system_tasks(
fulfilled_nodes, job_type_resources, when)
job_exe_count = 0
if sys_tasks_scheduled:
# Only schedule new job executions if all needed system tasks have been scheduled
job_type_limits = self._calculate_job_type_limits(
job_types, running_job_exes)
job_exe_count = self._schedule_new_job_exes(
framework_id, fulfilled_nodes, job_types, job_type_limits,
job_type_resources, workspaces)
else:
# TODO: this is a good place for a scheduler warning in the status JSON
logger.warning('No new jobs scheduled due to waiting system tasks')
if framework_id != scheduler_mgr.framework_id:
logger.warning(
'Scheduler framework ID changed, skipping task launch')
return 0
self._allocate_offers(nodes)
task_count, offer_count = self._launch_tasks(driver, nodes)
scheduler_mgr.add_scheduling_counts(job_exe_count, task_count,
offer_count)
return task_count
