def create_driver(framework_name, scheduler, system_paasta_config, implicit_acks=False): framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = framework_name framework.failover_timeout = 604800 framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name) framework.checkpoint = True credential = mesos_pb2.Credential() credential.principal = system_paasta_config.get_paasta_native_config( )['principal'] credential.secret = system_paasta_config.get_paasta_native_config( )['secret'] framework.principal = system_paasta_config.get_paasta_native_config( )['principal'] driver = MesosSchedulerDriver( scheduler, framework, '%s:%d' % (mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT), implicit_acks, credential) return driver
def main(master): logging.basicConfig(level=logging.INFO, format='[%(asctime)s %(levelname)s] %(message)s') # Create a new executor executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = 'ExampleExecutor' executor.name = executor.executor_id.value executor.command.value = os.path.abspath('./executor-skeleton.py') # Create a new framework framework = mesos_pb2.FrameworkInfo() framework.user = '' # the current user framework.name = 'ExampleFramework' framework.checkpoint = True implicitAcknowledgements = 1 if os.getenv('EXAMPLE_AUTHENTICATE'): logging.info('Enabling framework authentication') credential = mesos_pb2.Credential() credential.principal = os.getenv('EXAMPLE_PRINCIPAL') credential.secret = os.getenv('EXAMPLE_SECRET') framework.principal = os.getenv('EXAMPLE_PRINCIPAL') driver = MesosSchedulerDriver(ExampleScheduler(executor), framework, master, implicitAcknowledgements, credential) else: framework.principal = framework.name driver = MesosSchedulerDriver(ExampleScheduler(executor), framework, master, implicitAcknowledgements) def signal_handler(signal, frame): logging.info('Shutting down') driver.stop() # driver.run() blocks, so we run it in a separate thread. # This way, we can catch a SIGINT to kill the framework. def run_driver_thread(): status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 driver.stop() # Ensure the driver process terminates sys.exit(status) driver_thread = Thread(target=run_driver_thread, args=()) driver_thread.start() logging.info('Scheduler running, Ctrl-C to exit') signal.signal(signal.SIGINT, signal_handler) # Block the main thread while the driver thread is alive while driver_thread.is_alive(): time.sleep(1) logging.info('Framework finished.') sys.exit(0)
def mesos_credentials(cfg): """Establish credential information """ credentials = MesosPb2.Credential() credentials.principal = cfg.mesos.principal credentials.secret = cfg.mesos.secret return credentials
def run_scheduler(self, mesos_master): logger.info("I am the leader") self.scheduler = ScaleScheduler() self.scheduler.initialize() scheduler_mgr.hostname = socket.getfqdn() framework = mesos_pb2.FrameworkInfo() framework.user = '' # Have Mesos fill in the current user. framework.name = os.getenv('DCOS_PACKAGE_FRAMEWORK_NAME', 'Scale') webserver_address = os.getenv('SCALE_WEBSERVER_ADDRESS') if webserver_address: framework.webui_url = webserver_address logger.info('Connecting to Mesos master at %s', mesos_master) # TODO(vinod): Make checkpointing the default when it is default on the slave. if MESOS_CHECKPOINT: logger.info('Enabling checkpoint for the framework') framework.checkpoint = True if MESOS_AUTHENTICATE: logger.info('Enabling authentication for the framework') if not DEFAULT_PRINCIPLE: logger.error( 'Expecting authentication principal in the environment') sys.exit(1) if not DEFAULT_SECRET: logger.error( 'Expecting authentication secret in the environment') sys.exit(1) credential = mesos_pb2.Credential() credential.principal = DEFAULT_PRINCIPLE credential.secret = DEFAULT_SECRET self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master, credential) else: self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master) try: status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1 except: status = 1 logger.exception('Mesos Scheduler Driver returned an exception') #Perform a shut down and return any non-zero status shutdown_status = self._shutdown() status = status or shutdown_status logger.info('Exiting...') sys.exit(status)
def start_framework(master_uri, exe_path, cpu_alloc, mem_alloc, refuse_seconds): executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = "default" executor.command.value = exe_path executor.name = "QoSon Executor (Python)" executor.source = "network monitoring" framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = "QoSon Framework (Python)" implicitAcknowledgements = 1 if os.getenv("MESOS_EXPLICIT_ACKNOWLEDGEMENTS"): print "Enabling explicit status update acknowledgements" implicitAcknowledgements = 0 # create a scheduler and capture the command line options sched = NetmonScheduler(implicitAcknowledgements, executor, cpu_alloc, mem_alloc, refuse_seconds) if os.getenv("MESOS_CHECKPOINT"): print "Enabling checkpoint for the framework" framework.checkpoint = True if os.getenv("MESOS_AUTHENTICATE"): print "Enabling authentication for the framework" if not os.getenv("DEFAULT_PRINCIPAL"): print "Expecting authentication principal in the environment" sys.exit(1) if not os.getenv("DEFAULT_SECRET"): print "Expecting authentication secret in the environment" sys.exit(1) credential = mesos_pb2.Credential() credential.principal = os.getenv("DEFAULT_PRINCIPAL") credential.secret = os.getenv("DEFAULT_SECRET") framework.principal = os.getenv("DEFAULT_PRINCIPAL") driver = mesos.native.MesosSchedulerDriver( sched, framework, master_uri, implicitAcknowledgements, credential) else: framework.principal = "test-framework-python" driver = mesos.native.MesosSchedulerDriver( sched, framework, master_uri, implicitAcknowledgements) return driver, sched
def run_scheduler(self, mesos_master): logger.info("I am the leader") self.scheduler = ScaleScheduler() framework = mesos_pb2.FrameworkInfo() framework.user = '' # Have Mesos fill in the current user. framework.name = 'Scale' logger.info('Connecting to Mesos master at %s', mesos_master) # TODO(vinod): Make checkpointing the default when it is default on the slave. if MESOS_CHECKPOINT: logger.info('Enabling checkpoint for the framework') framework.checkpoint = True if MESOS_AUTHENTICATE: logger.info('Enabling authentication for the framework') if not DEFAULT_PRINCIPLE: logger.error( 'Expecting authentication principal in the environment') sys.exit(1) if not DEFAULT_SECRET: logger.error( 'Expecting authentication secret in the environment') sys.exit(1) credential = mesos_pb2.Credential() credential.principal = DEFAULT_PRINCIPLE credential.secret = DEFAULT_SECRET self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master, credential) else: self.driver = MesosSchedulerDriver(self.scheduler, framework, mesos_master) status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1 # Perform any required clean up operations like stopping background threads status = status or self._shutdown() logger.info('Exiting...') sys.exit(status)
def start(self): self.task_queue = Queue() self.result_queue = Queue() framework = mesos_pb2.FrameworkInfo() framework.user = '' if not configuration.get('mesos', 'MASTER'): self.log.error("Expecting mesos master URL for mesos executor") raise AirflowException( "mesos.master not provided for mesos executor") master = configuration.get('mesos', 'MASTER') framework.name = get_framework_name() if not configuration.get('mesos', 'TASK_CPU'): task_cpu = 1 else: task_cpu = configuration.getint('mesos', 'TASK_CPU') if not configuration.get('mesos', 'TASK_MEMORY'): task_memory = 256 else: task_memory = configuration.getint('mesos', 'TASK_MEMORY') if configuration.getboolean('mesos', 'CHECKPOINT'): framework.checkpoint = True if configuration.get('mesos', 'FAILOVER_TIMEOUT'): # Import here to work around a circular import error from airflow.models import Connection # Query the database to get the ID of the Mesos Framework, if available. conn_id = FRAMEWORK_CONNID_PREFIX + framework.name session = Session() connection = session.query(Connection).filter_by( conn_id=conn_id).first() if connection is not None: # Set the Framework ID to let the scheduler reconnect with running tasks. framework.id.value = connection.extra framework.failover_timeout = configuration.getint( 'mesos', 'FAILOVER_TIMEOUT') else: framework.checkpoint = False self.log.info( 'MesosFramework master : %s, name : %s, cpu : %s, mem : %s, checkpoint : %s', master, framework.name, str(task_cpu), str(task_memory), str(framework.checkpoint)) implicit_acknowledgements = 1 if configuration.getboolean('mesos', 'AUTHENTICATE'): if not configuration.get('mesos', 'DEFAULT_PRINCIPAL'): self.log.error( "Expecting authentication principal in the environment") raise AirflowException( "mesos.default_principal not provided in authenticated mode" ) if not configuration.get('mesos', 'DEFAULT_SECRET'): self.log.error( "Expecting authentication secret in the environment") raise AirflowException( "mesos.default_secret not provided in authenticated mode") credential = mesos_pb2.Credential() credential.principal = configuration.get('mesos', 'DEFAULT_PRINCIPAL') credential.secret = configuration.get('mesos', 'DEFAULT_SECRET') framework.principal = credential.principal driver = mesos.native.MesosSchedulerDriver( AirflowMesosScheduler(self.task_queue, self.result_queue, task_cpu, task_memory), framework, master, implicit_acknowledgements, credential) else: framework.principal = 'Airflow' driver = mesos.native.MesosSchedulerDriver( AirflowMesosScheduler(self.task_queue, self.result_queue, task_cpu, task_memory), framework, master, implicit_acknowledgements) self.mesos_driver = driver self.mesos_driver.start()
scheduler = TestScheduler(implicitAcknowledgements, executor) if os.getenv("MESOS_AUTHENTICATE"): logger.info("Enabling authentication for the framework") if not os.getenv("DEFAULT_PRINCIPAL"): logger.info( "Expecting authentication principal in the environment") sys.exit(1) if not os.getenv("DEFAULT_SECRET"): logger.info("Expecting authentication secret in the environment") sys.exit(1) credential = mesos_pb2.Credential() credential.principal = os.getenv("DEFAULT_PRINCIPAL") credential.secret = os.getenv("DEFAULT_SECRET") framework.principal = os.getenv("DEFAULT_PRINCIPAL") driver = mesos.native.MesosSchedulerDriver(scheduler, framework, sys.argv[1], implicitAcknowledgements, credential) else: framework.principal = "test-framework-python" driver = mesos.native.MesosSchedulerDriver(scheduler, framework, sys.argv[1], implicitAcknowledgements)
def run_forever(conf): """Entrypoint to keep the framework running until terminated.""" logger.info('******************Start************') logger.debug('DB connection: %s', conf.l2_db_con) logger.debug("Minimum Senes Per Seg: %s", conf.minscenesperseg) logger.debug('Segment query: %s', conf.segment_query) global shutdown db.reset_records(db.connect(conf.l2_db_con)) # Establish framework, executor, and authentication credentials framework = mesos_pb2.FrameworkInfo() framework.user = conf.framework_user framework.name = "ARD Tile Framework" framework.principal = conf.mesos_principal framework.role = conf.mesos_role executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = "default" executor.name = "ARD Tile executor" implicit_acks = 1 scheduler = ArdTileScheduler(implicit_acks, executor, conf) if not conf.disable_creds: logger.info(" MESOS creds ENABLED") credential = mesos_pb2.Credential() credential.principal = conf.mesos_principal credential.secret = conf.mesos_secret driver = mesos.native.MesosSchedulerDriver(scheduler, framework, conf.master, implicit_acks, credential) else: logger.info(" MESOS creds disabled") driver = mesos.native.MesosSchedulerDriver(scheduler, framework, conf.master, implicit_acks) shutdown = Shutdown() def run_driver_async(): """Thread for async communication with Mesos offers.""" # driver.run() blocks, so run it in a separate thread. status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 driver.stop() sys.exit(status) framework_thread = Thread(target=run_driver_async, args=()) framework_thread.start() while framework_thread.is_alive(): # If a shutdown has been requested, suppress offers and wait for the # framework thread to complete. if shutdown.flag: logger.info("Shutdown requested....") driver.suppressOffers() while framework_thread.is_alive(): logger.debug("Thread alive, sleep 5....") time.sleep(5) break # If the job queue is empty, get work. if (not scheduler.jobs and queue_segments(scheduler.jobs, conf, db.connect(conf.l2_db_con)) == ERROR): driver.stop(True) sys.exit(1) # If there's no new work to be done or the max number of jobs are # already running, suppress offers and wait for some jobs to finish. if (not scheduler.jobs or not scheduler.scheduling_allowed()): logger.info("No jobs or scheduling not allowed....") driver.suppressOffers() while not scheduler.scheduling_allowed(): logger.debug("Scheduling not alive, sleep 20....") time.sleep(20) while not scheduler.jobs: if queue_segments(scheduler.jobs, conf, db.connect(conf.l2_db_con)) == ERROR: driver.stop(True) sys.exit(1) time.sleep(20) driver.reviveOffers()
def start(self): self.task_queue = Queue() self.result_queue = Queue() framework = mesos_pb2.FrameworkInfo() framework.user = '' if not conf.get('mesos', 'MASTER'): logging.error("Expecting mesos master URL for mesos executor") raise AirflowException( "mesos.master not provided for mesos executor") master = conf.get('mesos', 'MASTER') if not conf.get('mesos', 'FRAMEWORK_NAME'): framework.name = 'Airflow' else: framework.name = conf.get('mesos', 'FRAMEWORK_NAME') if not conf.get('mesos', 'TASK_CPU'): task_cpu = 1 else: task_cpu = conf.getint('mesos', 'TASK_CPU') if not conf.get('mesos', 'TASK_MEMORY'): task_memory = 256 else: task_memory = conf.getint('mesos', 'TASK_MEMORY') if (conf.getboolean('mesos', 'CHECKPOINT')): framework.checkpoint = True else: framework.checkpoint = False logging.info( 'MesosFramework master : %s, name : %s, cpu : %s, mem : %s, checkpoint : %s', master, framework.name, str(task_cpu), str(task_memory), str(framework.checkpoint)) implicit_acknowledgements = 1 if (conf.getboolean('mesos', 'AUTHENTICATE')): if not conf.get('mesos', 'DEFAULT_PRINCIPAL'): logging.error( "Expecting authentication principal in the environment") raise AirflowException( "mesos.default_principal not provided in authenticated mode" ) if not conf.get('mesos', 'DEFAULT_SECRET'): logging.error( "Expecting authentication secret in the environment") raise AirflowException( "mesos.default_secret not provided in authenticated mode") credential = mesos_pb2.Credential() credential.principal = conf.get('mesos', 'DEFAULT_PRINCIPAL') credential.secret = conf.get('mesos', 'DEFAULT_SECRET') framework.principal = credential.principal driver = mesos.native.MesosSchedulerDriver( AirflowMesosScheduler(self.task_queue, self.result_queue, task_cpu, task_memory), framework, master, implicit_acknowledgements, credential) else: framework.principal = 'Airflow' driver = mesos.native.MesosSchedulerDriver( AirflowMesosScheduler(self.task_queue, self.result_queue, task_cpu, task_memory), framework, master, implicit_acknowledgements) self.mesos_driver = driver self.mesos_driver.start()