def create_driver(framework_name,
scheduler,
system_paasta_config,
implicit_acks=False):
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = framework_name
framework.failover_timeout = 604800
framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name)
framework.checkpoint = True
credential = mesos_pb2.Credential()
credential.principal = system_paasta_config.get_paasta_native_config(
)['principal']
credential.secret = system_paasta_config.get_paasta_native_config(
)['secret']
framework.principal = system_paasta_config.get_paasta_native_config(
)['principal']
driver = MesosSchedulerDriver(
scheduler, framework, '%s:%d' %
(mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT),
implicit_acks, credential)
return driver
def main(master):
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s %(levelname)s] %(message)s')
# Create a new executor
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'ExampleExecutor'
executor.name = executor.executor_id.value
executor.command.value = os.path.abspath('./executor-skeleton.py')
# Create a new framework
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # the current user
framework.name = 'ExampleFramework'
framework.checkpoint = True
implicitAcknowledgements = 1
if os.getenv('EXAMPLE_AUTHENTICATE'):
logging.info('Enabling framework authentication')
credential = mesos_pb2.Credential()
credential.principal = os.getenv('EXAMPLE_PRINCIPAL')
credential.secret = os.getenv('EXAMPLE_SECRET')
framework.principal = os.getenv('EXAMPLE_PRINCIPAL')
driver = MesosSchedulerDriver(ExampleScheduler(executor), framework,
master, implicitAcknowledgements,
credential)
else:
framework.principal = framework.name
driver = MesosSchedulerDriver(ExampleScheduler(executor), framework,
master, implicitAcknowledgements)
def signal_handler(signal, frame):
logging.info('Shutting down')
driver.stop()
# driver.run() blocks, so we run it in a separate thread.
# This way, we can catch a SIGINT to kill the framework.
def run_driver_thread():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop() # Ensure the driver process terminates
sys.exit(status)
driver_thread = Thread(target=run_driver_thread, args=())
driver_thread.start()
logging.info('Scheduler running, Ctrl-C to exit')
signal.signal(signal.SIGINT, signal_handler)
# Block the main thread while the driver thread is alive
while driver_thread.is_alive():
time.sleep(1)
logging.info('Framework finished.')
sys.exit(0)
def mesos_credentials(cfg):
"""Establish credential information
"""
credentials = MesosPb2.Credential()
credentials.principal = cfg.mesos.principal
credentials.secret = cfg.mesos.secret
return credentials
def run_scheduler(self, mesos_master):
logger.info("I am the leader")
self.scheduler = ScaleScheduler()
self.scheduler.initialize()
scheduler_mgr.hostname = socket.getfqdn()
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # Have Mesos fill in the current user.
framework.name = os.getenv('DCOS_PACKAGE_FRAMEWORK_NAME', 'Scale')
webserver_address = os.getenv('SCALE_WEBSERVER_ADDRESS')
if webserver_address:
framework.webui_url = webserver_address
logger.info('Connecting to Mesos master at %s', mesos_master)
# TODO(vinod): Make checkpointing the default when it is default on the slave.
if MESOS_CHECKPOINT:
logger.info('Enabling checkpoint for the framework')
framework.checkpoint = True
if MESOS_AUTHENTICATE:
logger.info('Enabling authentication for the framework')
if not DEFAULT_PRINCIPLE:
logger.error(
'Expecting authentication principal in the environment')
sys.exit(1)
if not DEFAULT_SECRET:
logger.error(
'Expecting authentication secret in the environment')
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = DEFAULT_PRINCIPLE
credential.secret = DEFAULT_SECRET
self.driver = MesosSchedulerDriver(self.scheduler, framework,
mesos_master, credential)
else:
self.driver = MesosSchedulerDriver(self.scheduler, framework,
mesos_master)
try:
status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1
except:
status = 1
logger.exception('Mesos Scheduler Driver returned an exception')
#Perform a shut down and return any non-zero status
shutdown_status = self._shutdown()
status = status or shutdown_status
logger.info('Exiting...')
sys.exit(status)
def start_framework(master_uri, exe_path, cpu_alloc, mem_alloc,
refuse_seconds):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.command.value = exe_path
executor.name = "QoSon Executor (Python)"
executor.source = "network monitoring"
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "QoSon Framework (Python)"
implicitAcknowledgements = 1
if os.getenv("MESOS_EXPLICIT_ACKNOWLEDGEMENTS"):
print "Enabling explicit status update acknowledgements"
implicitAcknowledgements = 0
# create a scheduler and capture the command line options
sched = NetmonScheduler(implicitAcknowledgements, executor, cpu_alloc,
mem_alloc, refuse_seconds)
if os.getenv("MESOS_CHECKPOINT"):
print "Enabling checkpoint for the framework"
framework.checkpoint = True
if os.getenv("MESOS_AUTHENTICATE"):
print "Enabling authentication for the framework"
if not os.getenv("DEFAULT_PRINCIPAL"):
print "Expecting authentication principal in the environment"
sys.exit(1)
if not os.getenv("DEFAULT_SECRET"):
print "Expecting authentication secret in the environment"
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = os.getenv("DEFAULT_PRINCIPAL")
credential.secret = os.getenv("DEFAULT_SECRET")
framework.principal = os.getenv("DEFAULT_PRINCIPAL")
driver = mesos.native.MesosSchedulerDriver(
sched, framework, master_uri, implicitAcknowledgements,
credential)
else:
framework.principal = "test-framework-python"
driver = mesos.native.MesosSchedulerDriver(
sched, framework, master_uri, implicitAcknowledgements)
return driver, sched
def run_scheduler(self, mesos_master):
logger.info("I am the leader")
self.scheduler = ScaleScheduler()
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # Have Mesos fill in the current user.
framework.name = 'Scale'
logger.info('Connecting to Mesos master at %s', mesos_master)
# TODO(vinod): Make checkpointing the default when it is default on the slave.
if MESOS_CHECKPOINT:
logger.info('Enabling checkpoint for the framework')
framework.checkpoint = True
if MESOS_AUTHENTICATE:
logger.info('Enabling authentication for the framework')
if not DEFAULT_PRINCIPLE:
logger.error(
'Expecting authentication principal in the environment')
sys.exit(1)
if not DEFAULT_SECRET:
logger.error(
'Expecting authentication secret in the environment')
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = DEFAULT_PRINCIPLE
credential.secret = DEFAULT_SECRET
self.driver = MesosSchedulerDriver(self.scheduler, framework,
mesos_master, credential)
else:
self.driver = MesosSchedulerDriver(self.scheduler, framework,
mesos_master)
status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1
# Perform any required clean up operations like stopping background threads
status = status or self._shutdown()
logger.info('Exiting...')
sys.exit(status)
def start(self):
self.task_queue = Queue()
self.result_queue = Queue()
framework = mesos_pb2.FrameworkInfo()
framework.user = ''
if not configuration.get('mesos', 'MASTER'):
self.log.error("Expecting mesos master URL for mesos executor")
raise AirflowException(
"mesos.master not provided for mesos executor")
master = configuration.get('mesos', 'MASTER')
framework.name = get_framework_name()
if not configuration.get('mesos', 'TASK_CPU'):
task_cpu = 1
else:
task_cpu = configuration.getint('mesos', 'TASK_CPU')
if not configuration.get('mesos', 'TASK_MEMORY'):
task_memory = 256
else:
task_memory = configuration.getint('mesos', 'TASK_MEMORY')
if configuration.getboolean('mesos', 'CHECKPOINT'):
framework.checkpoint = True
if configuration.get('mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Query the database to get the ID of the Mesos Framework, if available.
conn_id = FRAMEWORK_CONNID_PREFIX + framework.name
session = Session()
connection = session.query(Connection).filter_by(
conn_id=conn_id).first()
if connection is not None:
# Set the Framework ID to let the scheduler reconnect with running tasks.
framework.id.value = connection.extra
framework.failover_timeout = configuration.getint(
'mesos', 'FAILOVER_TIMEOUT')
else:
framework.checkpoint = False
self.log.info(
'MesosFramework master : %s, name : %s, cpu : %s, mem : %s, checkpoint : %s',
master, framework.name, str(task_cpu), str(task_memory),
str(framework.checkpoint))
implicit_acknowledgements = 1
if configuration.getboolean('mesos', 'AUTHENTICATE'):
if not configuration.get('mesos', 'DEFAULT_PRINCIPAL'):
self.log.error(
"Expecting authentication principal in the environment")
raise AirflowException(
"mesos.default_principal not provided in authenticated mode"
)
if not configuration.get('mesos', 'DEFAULT_SECRET'):
self.log.error(
"Expecting authentication secret in the environment")
raise AirflowException(
"mesos.default_secret not provided in authenticated mode")
credential = mesos_pb2.Credential()
credential.principal = configuration.get('mesos',
'DEFAULT_PRINCIPAL')
credential.secret = configuration.get('mesos', 'DEFAULT_SECRET')
framework.principal = credential.principal
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements, credential)
else:
framework.principal = 'Airflow'
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements)
self.mesos_driver = driver
self.mesos_driver.start()
scheduler = TestScheduler(implicitAcknowledgements, executor)
if os.getenv("MESOS_AUTHENTICATE"):
logger.info("Enabling authentication for the framework")
if not os.getenv("DEFAULT_PRINCIPAL"):
logger.info(
"Expecting authentication principal in the environment")
sys.exit(1)
if not os.getenv("DEFAULT_SECRET"):
logger.info("Expecting authentication secret in the environment")
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = os.getenv("DEFAULT_PRINCIPAL")
credential.secret = os.getenv("DEFAULT_SECRET")
framework.principal = os.getenv("DEFAULT_PRINCIPAL")
driver = mesos.native.MesosSchedulerDriver(scheduler, framework,
sys.argv[1],
implicitAcknowledgements,
credential)
else:
framework.principal = "test-framework-python"
driver = mesos.native.MesosSchedulerDriver(scheduler, framework,
sys.argv[1],
implicitAcknowledgements)
def run_forever(conf):
"""Entrypoint to keep the framework running until terminated."""
logger.info('******************Start************')
logger.debug('DB connection: %s', conf.l2_db_con)
logger.debug("Minimum Senes Per Seg: %s", conf.minscenesperseg)
logger.debug('Segment query: %s', conf.segment_query)
global shutdown
db.reset_records(db.connect(conf.l2_db_con))
# Establish framework, executor, and authentication credentials
framework = mesos_pb2.FrameworkInfo()
framework.user = conf.framework_user
framework.name = "ARD Tile Framework"
framework.principal = conf.mesos_principal
framework.role = conf.mesos_role
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.name = "ARD Tile executor"
implicit_acks = 1
scheduler = ArdTileScheduler(implicit_acks, executor, conf)
if not conf.disable_creds:
logger.info(" MESOS creds ENABLED")
credential = mesos_pb2.Credential()
credential.principal = conf.mesos_principal
credential.secret = conf.mesos_secret
driver = mesos.native.MesosSchedulerDriver(scheduler, framework,
conf.master, implicit_acks,
credential)
else:
logger.info(" MESOS creds disabled")
driver = mesos.native.MesosSchedulerDriver(scheduler, framework,
conf.master, implicit_acks)
shutdown = Shutdown()
def run_driver_async():
"""Thread for async communication with Mesos offers."""
# driver.run() blocks, so run it in a separate thread.
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
framework_thread = Thread(target=run_driver_async, args=())
framework_thread.start()
while framework_thread.is_alive():
# If a shutdown has been requested, suppress offers and wait for the
# framework thread to complete.
if shutdown.flag:
logger.info("Shutdown requested....")
driver.suppressOffers()
while framework_thread.is_alive():
logger.debug("Thread alive, sleep 5....")
time.sleep(5)
break
# If the job queue is empty, get work.
if (not scheduler.jobs
and queue_segments(scheduler.jobs, conf,
db.connect(conf.l2_db_con)) == ERROR):
driver.stop(True)
sys.exit(1)
# If there's no new work to be done or the max number of jobs are
# already running, suppress offers and wait for some jobs to finish.
if (not scheduler.jobs or not scheduler.scheduling_allowed()):
logger.info("No jobs or scheduling not allowed....")
driver.suppressOffers()
while not scheduler.scheduling_allowed():
logger.debug("Scheduling not alive, sleep 20....")
time.sleep(20)
while not scheduler.jobs:
if queue_segments(scheduler.jobs, conf,
db.connect(conf.l2_db_con)) == ERROR:
driver.stop(True)
sys.exit(1)
time.sleep(20)
driver.reviveOffers()
def start(self):
self.task_queue = Queue()
self.result_queue = Queue()
framework = mesos_pb2.FrameworkInfo()
framework.user = ''
if not conf.get('mesos', 'MASTER'):
logging.error("Expecting mesos master URL for mesos executor")
raise AirflowException(
"mesos.master not provided for mesos executor")
master = conf.get('mesos', 'MASTER')
if not conf.get('mesos', 'FRAMEWORK_NAME'):
framework.name = 'Airflow'
else:
framework.name = conf.get('mesos', 'FRAMEWORK_NAME')
if not conf.get('mesos', 'TASK_CPU'):
task_cpu = 1
else:
task_cpu = conf.getint('mesos', 'TASK_CPU')
if not conf.get('mesos', 'TASK_MEMORY'):
task_memory = 256
else:
task_memory = conf.getint('mesos', 'TASK_MEMORY')
if (conf.getboolean('mesos', 'CHECKPOINT')):
framework.checkpoint = True
else:
framework.checkpoint = False
logging.info(
'MesosFramework master : %s, name : %s, cpu : %s, mem : %s, checkpoint : %s',
master, framework.name, str(task_cpu), str(task_memory),
str(framework.checkpoint))
implicit_acknowledgements = 1
if (conf.getboolean('mesos', 'AUTHENTICATE')):
if not conf.get('mesos', 'DEFAULT_PRINCIPAL'):
logging.error(
"Expecting authentication principal in the environment")
raise AirflowException(
"mesos.default_principal not provided in authenticated mode"
)
if not conf.get('mesos', 'DEFAULT_SECRET'):
logging.error(
"Expecting authentication secret in the environment")
raise AirflowException(
"mesos.default_secret not provided in authenticated mode")
credential = mesos_pb2.Credential()
credential.principal = conf.get('mesos', 'DEFAULT_PRINCIPAL')
credential.secret = conf.get('mesos', 'DEFAULT_SECRET')
framework.principal = credential.principal
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements, credential)
else:
framework.principal = 'Airflow'
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements)
self.mesos_driver = driver
self.mesos_driver.start()
