class Running(object): def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'), implicit_acknowledge=1, *args, **kwargs): scheduler = SchedulerProxy(scheduler) framework = FrameworkInfo(name=name, user=user, *args, **kwargs) self.driver = MesosSchedulerDriver(scheduler, encode(framework), master, implicit_acknowledge) def shutdown(signal, frame): self.driver.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.driver.stop) def run(self): return self.driver.run() def start(self): status = self.driver.start() assert status == mesos_pb2.DRIVER_RUNNING return status def stop(self): logging.info("Stopping Mesos driver") self.driver.stop() logging.info("Joining Mesos driver") result = self.driver.join() logging.info("Joined Mesos driver") if result != mesos_pb2.DRIVER_STOPPED: raise RuntimeError("Mesos driver failed with %i", result) def join(self): return self.driver.join() def __enter__(self): self.start() return self def __exit__(self, type, value, traceback): self.stop()
class Running(object): def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'), implicit_acknowledge=1, *args, **kwargs): scheduler = SchedulerProxy(scheduler) framework = FrameworkInfo(name=name, user=user, *args, **kwargs) self.driver = MesosSchedulerDriver(scheduler, encode(framework), master, implicit_acknowledge) def shutdown(signal, frame): self.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.stop) def run(self): return self.driver.run() def start(self): status = self.driver.start() assert status == mesos_pb2.DRIVER_RUNNING return status def stop(self): return self.driver.stop() def join(self): return self.driver.join() def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_value, traceback): self.stop() self.join() if exc_type: raise exc_type, exc_value, traceback
class Running(object): def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'), implicit_acknowledge=1, *args, **kwargs): framework = FrameworkInfo(name=name, user=user, *args, **kwargs) scheduler = SchedulerProxy(scheduler) self.driver = MesosSchedulerDriver(scheduler, encode(framework), master, implicit_acknowledge) def shutdown(signal, frame): self.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.stop) def run(self): return self.driver.run() def start(self): status = self.driver.start() assert status == mesos_pb2.DRIVER_RUNNING return status def stop(self): return self.driver.stop() def join(self): return self.driver.join() def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_value, traceback): self.stop() self.join() if exc_type: raise exc_type, exc_value, traceback
class Command(BaseCommand): """Command that launches the Scale scheduler """ help = 'Launches the Scale scheduler' def add_arguments(self, parser): parser.add_argument('-m', '--master', action='store', default=settings.MESOS_MASTER, help='The master to connect to') def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the scheduler. """ # Register a listener to handle clean shutdowns signal.signal(signal.SIGTERM, self._onsigterm) # TODO: clean this up mesos_master = options.get('master') logger.info('Scale Scheduler %s', settings.VERSION) try: scheduler_zk = settings.SCHEDULER_ZK except: scheduler_zk = None if scheduler_zk is not None: import socket from scheduler import cluster_utils my_id = socket.gethostname() cluster_utils.wait_for_leader(scheduler_zk, my_id, self.run_scheduler, mesos_master) else: # leader election is disabled self.run_scheduler(mesos_master) def run_scheduler(self, mesos_master): logger.info("I am the leader") self.scheduler = ScaleScheduler() framework = mesos_pb2.FrameworkInfo() framework.user = '' # Have Mesos fill in the current user. framework.name = os.getenv('DCOS_PACKAGE_FRAMEWORK_NAME', 'Scale') webserver_address = os.getenv('SCALE_WEBSERVER_ADDRESS') if webserver_address: framework.webui_url = webserver_address logger.info('Connecting to Mesos master at %s', mesos_master) # TODO(vinod): Make checkpointing the default when it is default on the slave. if MESOS_CHECKPOINT: logger.info('Enabling checkpoint for the framework') framework.checkpoint = True if MESOS_AUTHENTICATE: logger.info('Enabling authentication for the framework') if not DEFAULT_PRINCIPLE: logger.error( 'Expecting authentication principal in the environment') sys.exit(1) if not DEFAULT_SECRET: logger.error( 'Expecting authentication secret in the environment') sys.exit(1) credential = mesos_pb2.Credential() credential.principal = DEFAULT_PRINCIPLE credential.secret = DEFAULT_SECRET self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master, credential) else: self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master) try: status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1 except: status = 1 logger.exception('Mesos Scheduler Driver returned an exception') #Perform a shut down and return any non-zero status shutdown_status = self._shutdown status = status or shutdown_status logger.info('Exiting...') sys.exit(status) def _onsigterm(self, signum, _frame): """See signal callback registration: :py:func:`signal.signal`. This callback performs a clean shutdown when a TERM signal is received. """ logger.info('Scheduler command terminated due to signal: %i', signum) self._shutdown() sys.exit(1) def _shutdown(self): """Performs any clean up required by this command. :returns: The exit status code based on whether the shutdown operation was clean with no exceptions. :rtype: int """ status = 0 try: if self.scheduler: self.scheduler.shutdown() except: logger.exception('Failed to properly shutdown Scale scheduler.') status = 1 try: if self.driver: self.driver.stop() except: logger.exception('Failed to properly stop Mesos driver.') status = 1 return status
task = mesos_pb2.TaskInfo() id = uuid.uuid4() task.task_id.value = str(id) task.slave_id.value = offer.slave_id.value task.name = "task {}".format(str(id)) cpus = task.resources.add() cpus.name = "cpus" cpus.type = mesos_pb2.Value.SCALAR cpus.scalar.value = 0.1 mem = task.resources.add() mem.name = "mem" mem.type = mesos_pb2.Value.SCALAR mem.scalar.value = 32 return task if __name__ == '__main__': log("XXX framework started") framework = mesos_pb2.FrameworkInfo() framework.user = "******" framework.name = "nixos-test-framework" driver = MesosSchedulerDriver( NixosTestScheduler(), framework, sys.argv[1] + ":5050" ) driver.run()
class Command(BaseCommand): """Command that launches the Scale scheduler """ option_list = BaseCommand.option_list + ( make_option('-m', '--master', action='store', type='str', default=settings.MESOS_MASTER, help=('The master to connect to')), ) help = 'Launches the Scale scheduler' def handle(self, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the scheduler. """ # Register a listener to handle clean shutdowns signal.signal(signal.SIGTERM, self._onsigterm) # TODO: clean this up mesos_master = options.get('master') logger.info('Scale Scheduler %s', settings.VERSION) try: scheduler_zk = settings.SCHEDULER_ZK except: scheduler_zk = None if scheduler_zk is not None: import socket from scheduler import cluster_utils my_id = socket.gethostname() cluster_utils.wait_for_leader(scheduler_zk, my_id, self.run_scheduler, mesos_master) else: # leader election is disabled self.run_scheduler(mesos_master) def run_scheduler(self, mesos_master): logger.info("I am the leader") self.scheduler = ScaleScheduler() framework = mesos_pb2.FrameworkInfo() framework.user = '' # Have Mesos fill in the current user. framework.name = 'Scale' logger.info('Connecting to Mesos master at %s', mesos_master) # TODO(vinod): Make checkpointing the default when it is default on the slave. if MESOS_CHECKPOINT: logger.info('Enabling checkpoint for the framework') framework.checkpoint = True if MESOS_AUTHENTICATE: logger.info('Enabling authentication for the framework') if not DEFAULT_PRINCIPLE: logger.error('Expecting authentication principal in the environment') sys.exit(1) if not DEFAULT_SECRET: logger.error('Expecting authentication secret in the environment') sys.exit(1) credential = mesos_pb2.Credential() credential.principal = DEFAULT_PRINCIPLE credential.secret = DEFAULT_SECRET self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master, credential) else: self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master) try: status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1 except: status = 1 logger.exception('Mesos Scheduler Driver returned an exception') #Perform a shut down and return any non-zero status shutdown_status = self._shutdown status = status or shutdown_status logger.info('Exiting...') sys.exit(status) def _onsigterm(self, signum, _frame): """See signal callback registration: :py:func:`signal.signal`. This callback performs a clean shutdown when a TERM signal is received. """ logger.info('Scheduler command terminated due to signal: %i', signum) self._shutdown() sys.exit(1) def _shutdown(self): """Performs any clean up required by this command. :returns: The exit status code based on whether the shutdown operation was clean with no exceptions. :rtype: int """ status = 0 try: if self.scheduler: self.scheduler.shutdown() except: logger.exception('Failed to properly shutdown Scale scheduler.') status = 1 try: if self.driver: self.driver.stop() except: logger.exception('Failed to properly stop Mesos driver.') status = 1 return status
except Exception as e: raise e finally: mesos_lock.release() db.rhino_tasks.update({'name': task['name']}, {'$set': {'state': 'KILLED'}}) kill_those_that_depend_on(task['name']) kill_those_that_depend_on(doc['name']) db.rhino_tasks.update({'mesos_id': status.task_id.value}, {'$set': {'state': state, 'retCode': ret_code, 'message': message}}) if __name__ == '__main__': try: web_server_thread = threading.Thread(target=web_server, args=()) web_server_thread.start() framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = "rhino" mesos_driver = MesosSchedulerDriver( AppsomaRhinoScheduler(), framework, "zk://" + config['zookeeper_hosts'] + "/mesos" ) mesos_driver.run() except KeyboardInterrupt: print "KeyboardInterrupt" os.kill(os.getpid(), 9)
class Command(BaseCommand): '''Command that launches the Scale scheduler ''' option_list = BaseCommand.option_list + ( make_option('-m', '--master', action='store', type='str', default=settings.MESOS_MASTER, help=('The master to connect to')), ) help = 'Launches the Scale scheduler' def handle(self, **options): '''See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the scheduler. ''' # Register a listener to handle clean shutdowns signal.signal(signal.SIGTERM, self._onsigterm) # TODO: clean this up mesos_master = options.get('master') logger.info(u'Command starting: scale_scheduler') logger.info(u' - Master: %s', mesos_master) executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = 'scale' executor.command.value = '%s %s scale_executor' % (settings.PYTHON_EXECUTABLE, settings.MANAGE_FILE) executor.name = 'Scale Executor (Python)' self.scheduler = ScaleScheduler(executor) framework = mesos_pb2.FrameworkInfo() framework.user = '' # Have Mesos fill in the current user. framework.name = 'Scale Framework (Python)' # TODO(vinod): Make checkpointing the default when it is default on the slave. if MESOS_CHECKPOINT: logger.info('Enabling checkpoint for the framework') framework.checkpoint = True if MESOS_AUTHENTICATE: logger.info('Enabling authentication for the framework') if not DEFAULT_PRINCIPLE: logger.error('Expecting authentication principal in the environment') sys.exit(1) if not DEFAULT_SECRET: logger.error('Expecting authentication secret in the environment') sys.exit(1) credential = mesos_pb2.Credential() credential.principal = DEFAULT_PRINCIPLE credential.secret = DEFAULT_SECRET self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master, credential) else: self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master) status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1 # Perform any required clean up operations like stopping background threads status = status or self._shutdown() logger.info(u'Command completed: scale_scheduler') sys.exit(status) def _onsigterm(self, signum, _frame): '''See signal callback registration: :py:func:`signal.signal`. This callback performs a clean shutdown when a TERM signal is received. ''' logger.info(u'Scheduler command terminated due to signal: %i', signum) self._shutdown() sys.exit(1) def _shutdown(self): '''Performs any clean up required by this command. :returns: The exit status code based on whether the shutdown operation was clean with no exceptions. :rtype: int ''' status = 0 try: if self.scheduler: self.scheduler.shutdown() except: logger.exception('Failed to properly shutdown scale scheduler.') status = 1 try: if self.driver: self.driver.stop() except: logger.exception('Failed to properly stop Mesos driver.') status = 1 return status
elif update.task_id.value == "1": if update.message == "Command terminated with signal Killed: 9": print "#### Passed forced shutdown" else: print "!!!! Failed forced shutdown" self.tasksFinished += 1 if self.tasksFinished == TOTAL_TASKS: driver.stop() if __name__ == "__main__": if len(sys.argv) != 2: print "Usage: %s master" % sys.argv[0] sys.exit(1) framework = mesos_pb2.FrameworkInfo() framework.user = "" framework.name = "Escalation Framework (Python)" driver = MesosSchedulerDriver( EscalationScheduler(), framework, sys.argv[1]) status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 # Ensure that the driver process terminates. driver.stop(); sys.exit(status)
mem.type = mesos_pb2.Value.SCALAR mem.scalar.value = 1 time.sleep(2) logging.info("Launching task {task} " "using offer {offer}.".format(task=task.task_id.value, offer=offer.id.value)) tasks = [task] driver.launchTasks(offer.id, tasks) else: driver.stop() if __name__ == '__main__': # make us a framework framework = mesos_pb2.FrameworkInfo() framework.user = "******" # Have Mesos fill in the current user. framework.name = "hello-world" framework.checkpoint = True driver = MesosSchedulerDriver( HelloWorldScheduler(), framework, "127.0.0.1:5050/" # assumes running on the master ) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)