def registered(self, driver, frameworkId, masterInfo):
'''
Invoked when the scheduler successfully registers with a Mesos master.
It is called with the frameworkId, a unique ID generated by the
master, and the masterInfo which is information about the master
itself.
See documentation for :meth:`mesos_api.mesos.Scheduler.registered`.
'''
if self.debug:
connect_remote_debug()
logger.info('Scale scheduler registered as framework %s with Mesos master at %s:%i',
frameworkId.value, masterInfo.hostname, masterInfo.port)
self.driver = driver
self.framework_id = frameworkId.value
self.master_hostname = masterInfo.hostname
self.master_port = masterInfo.port
models.Scheduler.objects.update_master(self.master_hostname, self.master_port)
try:
initialize_system()
except Exception as ex:
logger.exception('Failed to perform system initialization, killing scheduler')
raise ex
try:
self._reconcile_running_jobs()
except Exception as ex:
logger.exception('Failed to query running jobs for reconciliation, killing scheduler')
raise ex
def registered(self, driver, frameworkId, masterInfo):
"""
Invoked when the scheduler successfully registers with a Mesos master.
It is called with the frameworkId, a unique ID generated by the
master, and the masterInfo which is information about the master
itself.
See documentation for :meth:`mesos_api.mesos.Scheduler.registered`.
"""
self._driver = driver
self._framework_id = frameworkId.value
self._master_hostname = masterInfo.hostname
self._master_port = masterInfo.port
logger.info(
'Scale scheduler registered as framework %s with Mesos master at %s:%i',
self._framework_id, self._master_hostname, self._master_port)
initialize_system()
Scheduler.objects.update_master(self._master_hostname,
self._master_port)
# Initial database sync
self._job_type_manager.sync_with_database()
self._scheduler_manager.sync_with_database()
self._workspace_manager.sync_with_database()
# Start up background threads
self._db_sync_thread = DatabaseSyncThread(self._driver,
self._job_exe_manager,
self._job_type_manager,
self._node_manager,
self._scheduler_manager,
self._workspace_manager)
db_sync_thread = threading.Thread(target=self._db_sync_thread.run)
db_sync_thread.daemon = True
db_sync_thread.start()
self._recon_thread = ReconciliationThread(self._driver)
recon_thread = threading.Thread(target=self._recon_thread.run)
recon_thread.daemon = True
recon_thread.start()
self._scheduling_thread = SchedulingThread(
self._driver, self._job_exe_manager, self._job_type_manager,
self._node_manager, self._offer_manager, self._scheduler_manager,
self._workspace_manager)
scheduling_thread = threading.Thread(
target=self._scheduling_thread.run)
scheduling_thread.daemon = True
scheduling_thread.start()
self._reconcile_running_jobs()
def registered(self, driver, frameworkId, masterInfo):
"""
Invoked when the scheduler successfully registers with a Mesos master.
It is called with the frameworkId, a unique ID generated by the
master, and the masterInfo which is information about the master
itself.
See documentation for :meth:`mesos_api.mesos.Scheduler.registered`.
"""
self._driver = driver
self._framework_id = frameworkId.value
self._master_hostname = masterInfo.hostname
self._master_port = masterInfo.port
logger.info('Scale scheduler registered as framework %s with Mesos master at %s:%i',
self._framework_id, self._master_hostname, self._master_port)
initialize_system()
Scheduler.objects.update_master(self._master_hostname, self._master_port)
# Initial database sync
self._job_type_manager.sync_with_database()
self._scheduler_manager.sync_with_database()
self._workspace_manager.sync_with_database()
# Start up background threads
self._db_sync_thread = DatabaseSyncThread(self._driver, self._job_exe_manager, self._job_type_manager,
self._node_manager, self._scheduler_manager, self._workspace_manager)
db_sync_thread = threading.Thread(target=self._db_sync_thread.run)
db_sync_thread.daemon = True
db_sync_thread.start()
self._recon_thread = ReconciliationThread(self._driver)
recon_thread = threading.Thread(target=self._recon_thread.run)
recon_thread.daemon = True
recon_thread.start()
self._scheduling_thread = SchedulingThread(self._driver, self._framework_id, self._job_exe_manager,
self._job_type_manager, self._node_manager, self._offer_manager,
self._scheduler_manager, self._workspace_manager)
scheduling_thread = threading.Thread(target=self._scheduling_thread.run)
scheduling_thread.daemon = True
scheduling_thread.start()
self._status_thread = StatusUpdateThread(self._status_manager)
status_thread = threading.Thread(target=self._status_thread.run)
status_thread.daemon = True
status_thread.start()
self._reconcile_running_jobs()
def test_create_clock_job(self):
'''Tests creating the Scale clock job'''
clock_job_type = JobType.objects.get_clock_job_type()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
self.assertEqual(count, 0)
initialize_system()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
self.assertEqual(count, 1)
# Make sure it only creates one job
initialize_system()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
self.assertEqual(count, 1)
def test_create_clock_job(self):
"""Tests creating the Scale clock job"""
clock_job_type = JobType.objects.get_clock_job_type()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
self.assertEqual(count, 0)
initialize_system()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
self.assertEqual(count, 1)
# Make sure it only creates one job
initialize_system()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
self.assertEqual(count, 1)
def initialize(self):
"""Initializes the scheduler and gets it ready to connect to Mesos. This method should only ever be called once.
"""
initialize_system()
# Initial database sync
logger.info('Performing initial sync with Scale database')
logger.info('Retrieving errors...')
reset_error_cache()
logger.info('Retrieving job execution metrics...')
job_exe_mgr.init_with_database()
logger.info('Retrieving job types...')
job_type_mgr.sync_with_database()
logger.info('Retrieving workspaces...')
workspace_mgr.sync_with_database()
logger.info('Retrieving scheduler settings...')
scheduler_mgr.sync_with_database()
# Start up background threads
self._threads = []
logger.info('Starting up background threads')
self._messaging_thread = MessagingThread()
restart_msg = RestartScheduler()
restart_msg.when = now()
self._messaging_thread.add_initial_messages([restart_msg])
messaging_thread = threading.Thread(target=self._messaging_thread.run)
messaging_thread.daemon = True
messaging_thread.start()
self._threads.append(messaging_thread)
self._recon_thread = ReconciliationThread()
recon_thread = threading.Thread(target=self._recon_thread.run)
recon_thread.daemon = True
recon_thread.start()
self._threads.append(recon_thread)
self._scheduler_status_thread = SchedulerStatusThread()
scheduler_status_thread = threading.Thread(target=self._scheduler_status_thread.run)
scheduler_status_thread.daemon = True
scheduler_status_thread.start()
self._threads.append(scheduler_status_thread)
self._scheduling_thread = SchedulingThread(self._client)
scheduling_thread = threading.Thread(target=self._scheduling_thread.run)
scheduling_thread.daemon = True
scheduling_thread.start()
self._threads.append(scheduling_thread)
self._sync_thread = SyncThread(self._driver)
sync_thread = threading.Thread(target=self._sync_thread.run)
sync_thread.daemon = True
sync_thread.start()
self._threads.append(sync_thread)
self._task_handling_thread = TaskHandlingThread(self._driver)
task_handling_thread = threading.Thread(target=self._task_handling_thread.run)
task_handling_thread.daemon = True
task_handling_thread.start()
self._threads.append(task_handling_thread)
self._task_update_thread = TaskUpdateThread()
task_update_thread = threading.Thread(target=self._task_update_thread.run)
task_update_thread.daemon = True
task_update_thread.start()
self._threads.append(task_update_thread)
def registered(self, driver, frameworkId, masterInfo):
"""
Invoked when the scheduler successfully registers with a Mesos master.
It is called with the frameworkId, a unique ID generated by the
master, and the masterInfo which is information about the master
itself.
See documentation for :meth:`mesos_api.mesos.Scheduler.registered`.
"""
self._driver = driver
self._framework_id = frameworkId.value
self._master_hostname = masterInfo.hostname
self._master_port = masterInfo.port
logger.info(
'Scale scheduler registered as framework %s with Mesos master at %s:%i',
self._framework_id, self._master_hostname, self._master_port)
initialize_system()
Scheduler.objects.update_master(self._master_hostname,
self._master_port)
scheduler_mgr.update_from_mesos(
self._framework_id,
HostAddress(self._master_hostname, self._master_port))
recon_mgr.driver = self._driver
# Initial database sync
logger.info('Performing initial sync with Scale database')
reset_error_cache()
job_exe_mgr.init_with_database()
job_type_mgr.sync_with_database()
scheduler_mgr.sync_with_database()
workspace_mgr.sync_with_database()
# Start up background threads
self._messaging_thread = MessagingThread()
restart_msg = RestartScheduler()
restart_msg.when = now()
self._messaging_thread.add_initial_messages([restart_msg])
messaging_thread = threading.Thread(target=self._messaging_thread.run)
messaging_thread.daemon = True
messaging_thread.start()
self._recon_thread = ReconciliationThread()
recon_thread = threading.Thread(target=self._recon_thread.run)
recon_thread.daemon = True
recon_thread.start()
self._scheduler_status_thread = SchedulerStatusThread()
scheduler_status_thread = threading.Thread(
target=self._scheduler_status_thread.run)
scheduler_status_thread.daemon = True
scheduler_status_thread.start()
self._scheduling_thread = SchedulingThread(self._driver)
scheduling_thread = threading.Thread(
target=self._scheduling_thread.run)
scheduling_thread.daemon = True
scheduling_thread.start()
self._sync_thread = SyncThread(self._driver)
sync_thread = threading.Thread(target=self._sync_thread.run)
sync_thread.daemon = True
sync_thread.start()
self._task_handling_thread = TaskHandlingThread(self._driver)
task_handling_thread = threading.Thread(
target=self._task_handling_thread.run)
task_handling_thread.daemon = True
task_handling_thread.start()
self._task_update_thread = TaskUpdateThread()
task_update_thread = threading.Thread(
target=self._task_update_thread.run)
task_update_thread.daemon = True
task_update_thread.start()
self._reconcile_running_jobs()