def subscribed(self, driver): """ Invoked when the scheduler successfully registers with a Mesos master. It is called with a SchedulerDriver which has attributes that include the frameworkId, and the mesos_url, which indicates the master currently leading. """ self._driver = driver self._framework_id = driver.frameworkId self._master_host_address = host_address_from_mesos_url( driver.mesos_url) logger.info( 'Scale scheduler registered as framework %s with Mesos master at %s:%i', self._framework_id, self._master_host_address.hostname, self._master_host_address.port) scheduler_mgr.update_from_mesos(self._framework_id, self._master_host_address) # Update driver for background threads recon_mgr.driver = self._driver self._scheduling_thread.client = self._client self._sync_thread.driver = self._driver self._task_handling_thread.driver = self._driver self._reconcile_running_jobs()
def reregistered(self, driver, masterInfo): """ Invoked when the scheduler re-registers with a newly elected Mesos master. This is only called when the scheduler has previously been registered. masterInfo contains information about the newly elected master. See documentation for :meth:`mesos_api.mesos.Scheduler.reregistered`. """ self._driver = driver self._master_hostname = masterInfo.hostname self._master_port = masterInfo.port logger.info('Scale scheduler re-registered with Mesos master at %s:%i', self._master_hostname, self._master_port) Scheduler.objects.update_master(self._master_hostname, self._master_port) scheduler_mgr.update_from_mesos(mesos_address=HostAddress( self._master_hostname, self._master_port)) # Update driver for background threads recon_mgr.driver = self._driver self._scheduling_thread.driver = self._driver self._sync_thread.driver = self._driver self._task_handling_thread.driver = self._driver self._reconcile_running_jobs()
def registered(self, driver, frameworkId, masterInfo): """ Invoked when the scheduler successfully registers with a Mesos master. It is called with the frameworkId, a unique ID generated by the master, and the masterInfo which is information about the master itself. See documentation for :meth:`mesos_api.mesos.Scheduler.registered`. """ self._driver = driver self._framework_id = frameworkId.value self._master_hostname = masterInfo.hostname self._master_port = masterInfo.port logger.info( 'Scale scheduler registered as framework %s with Mesos master at %s:%i', self._framework_id, self._master_hostname, self._master_port) Scheduler.objects.update_master(self._master_hostname, self._master_port) scheduler_mgr.update_from_mesos( self._framework_id, HostAddress(self._master_hostname, self._master_port)) # Update driver for background threads recon_mgr.driver = self._driver self._scheduling_thread.driver = self._driver self._sync_thread.driver = self._driver self._task_handling_thread.driver = self._driver self._reconcile_running_jobs()
def setUp(self): django.setup() reset_error_cache() self.framework_id = '1234' Scheduler.objects.initialize_scheduler() Scheduler.objects.update( num_message_handlers=0 ) # Prevent message handler tasks from scheduling self._client = MagicMock() scheduler_mgr.sync_with_database() scheduler_mgr.update_from_mesos(framework_id=self.framework_id) resource_mgr.clear() job_exe_mgr.clear() self.agent_1 = Agent('agent_1', 'host_1') self.agent_2 = Agent('agent_2', 'host_2') self.agent_3 = Agent('agent_3', 'host_2') node_mgr.clear() node_mgr.register_agents([self.agent_1, self.agent_2]) node_mgr.sync_with_database(scheduler_mgr.config) # Ignore initial cleanup, health check, and image pull tasks for node in node_mgr.get_nodes(): node._last_health_task = now() node._initial_cleanup_completed() node._is_image_pulled = True node._update_state() if node.agent_id == 'agent_1': self.node_1_id = node.id cleanup_mgr.update_nodes(node_mgr.get_nodes()) self.node_1 = Node.objects.get(id=self.node_1_id) # Ignore system tasks system_task_mgr._is_db_update_completed = True self.queue_1 = queue_test_utils.create_queue(cpus_required=4.0, mem_required=1024.0, disk_in_required=100.0, disk_out_required=200.0, disk_total_required=300.0) self.queue_2 = queue_test_utils.create_queue(cpus_required=8.0, mem_required=512.0, disk_in_required=400.0, disk_out_required=45.0, disk_total_required=445.0) self.queue_large = queue_test_utils.create_queue( resources=NodeResources([Cpus( 125.0), Mem(12048.0), Disk(12048.0)])) job_type_mgr.sync_with_database()
def reconnected(self, message): """ Invoked when the scheduler re-registers with a newly elected Mesos master. This is only called when the scheduler has previously been registered. """ self._framework_id = self._driver.frameworkId self._master_host_address = host_address_from_mesos_url(self._driver.mesos_url) logger.info('Scale scheduler re-registered with Mesos master at %s:%i', self._master_host_address.hostname, self._master_host_address.port) scheduler_mgr.update_from_mesos(mesos_address=self._master_host_address) self._reconcile_running_jobs()
def registered(self, driver, frameworkId, masterInfo): """ Invoked when the scheduler successfully registers with a Mesos master. It is called with the frameworkId, a unique ID generated by the master, and the masterInfo which is information about the master itself. See documentation for :meth:`mesos_api.mesos.Scheduler.registered`. """ self._driver = driver self._framework_id = frameworkId.value self._master_hostname = masterInfo.hostname self._master_port = masterInfo.port logger.info( 'Scale scheduler registered as framework %s with Mesos master at %s:%i', self._framework_id, self._master_hostname, self._master_port) initialize_system() Scheduler.objects.update_master(self._master_hostname, self._master_port) scheduler_mgr.update_from_mesos( self._framework_id, HostAddress(self._master_hostname, self._master_port)) recon_mgr.driver = self._driver # Initial database sync logger.info('Performing initial sync with Scale database') reset_error_cache() job_exe_mgr.init_with_database() job_type_mgr.sync_with_database() scheduler_mgr.sync_with_database() workspace_mgr.sync_with_database() # Start up background threads self._messaging_thread = MessagingThread() restart_msg = RestartScheduler() restart_msg.when = now() self._messaging_thread.add_initial_messages([restart_msg]) messaging_thread = threading.Thread(target=self._messaging_thread.run) messaging_thread.daemon = True messaging_thread.start() self._recon_thread = ReconciliationThread() recon_thread = threading.Thread(target=self._recon_thread.run) recon_thread.daemon = True recon_thread.start() self._scheduler_status_thread = SchedulerStatusThread() scheduler_status_thread = threading.Thread( target=self._scheduler_status_thread.run) scheduler_status_thread.daemon = True scheduler_status_thread.start() self._scheduling_thread = SchedulingThread(self._driver) scheduling_thread = threading.Thread( target=self._scheduling_thread.run) scheduling_thread.daemon = True scheduling_thread.start() self._sync_thread = SyncThread(self._driver) sync_thread = threading.Thread(target=self._sync_thread.run) sync_thread.daemon = True sync_thread.start() self._task_handling_thread = TaskHandlingThread(self._driver) task_handling_thread = threading.Thread( target=self._task_handling_thread.run) task_handling_thread.daemon = True task_handling_thread.start() self._task_update_thread = TaskUpdateThread() task_update_thread = threading.Thread( target=self._task_update_thread.run) task_update_thread.daemon = True task_update_thread.start() self._reconcile_running_jobs()