def main(master):
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s %(levelname)s] %(message)s')
# Create a new executor
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'ExampleExecutor'
executor.name = executor.executor_id.value
executor.command.value = os.path.abspath('./executor-skeleton.py')
# Create a new framework
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # the current user
framework.name = 'ExampleFramework'
framework.checkpoint = True
implicitAcknowledgements = 1
if os.getenv('EXAMPLE_AUTHENTICATE'):
logging.info('Enabling framework authentication')
credential = mesos_pb2.Credential()
credential.principal = os.getenv('EXAMPLE_PRINCIPAL')
credential.secret = os.getenv('EXAMPLE_SECRET')
framework.principal = os.getenv('EXAMPLE_PRINCIPAL')
driver = MesosSchedulerDriver(ExampleScheduler(executor), framework,
master, implicitAcknowledgements,
credential)
else:
framework.principal = framework.name
driver = MesosSchedulerDriver(ExampleScheduler(executor), framework,
master, implicitAcknowledgements)
def signal_handler(signal, frame):
logging.info('Shutting down')
driver.stop()
# driver.run() blocks, so we run it in a separate thread.
# This way, we can catch a SIGINT to kill the framework.
def run_driver_thread():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop() # Ensure the driver process terminates
sys.exit(status)
driver_thread = Thread(target=run_driver_thread, args=())
driver_thread.start()
logging.info('Scheduler running, Ctrl-C to exit')
signal.signal(signal.SIGINT, signal_handler)
# Block the main thread while the driver thread is alive
while driver_thread.is_alive():
time.sleep(1)
logging.info('Framework finished.')
sys.exit(0)
def run_scheduler(self, mesos_master):
logger.info("I am the leader")
self.scheduler = ScaleScheduler()
self.scheduler.initialize()
scheduler_mgr.hostname = socket.getfqdn()
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # Have Mesos fill in the current user.
framework.name = os.getenv('DCOS_PACKAGE_FRAMEWORK_NAME', 'Scale')
webserver_address = os.getenv('SCALE_WEBSERVER_ADDRESS')
if webserver_address:
framework.webui_url = webserver_address
logger.info('Connecting to Mesos master at %s', mesos_master)
# TODO(vinod): Make checkpointing the default when it is default on the slave.
if MESOS_CHECKPOINT:
logger.info('Enabling checkpoint for the framework')
framework.checkpoint = True
if MESOS_AUTHENTICATE:
logger.info('Enabling authentication for the framework')
if not DEFAULT_PRINCIPLE:
logger.error(
'Expecting authentication principal in the environment')
sys.exit(1)
if not DEFAULT_SECRET:
logger.error(
'Expecting authentication secret in the environment')
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = DEFAULT_PRINCIPLE
credential.secret = DEFAULT_SECRET
self.driver = MesosSchedulerDriver(self.scheduler, framework,
mesos_master, credential)
else:
self.driver = MesosSchedulerDriver(self.scheduler, framework,
mesos_master)
try:
status = 0 if self.driver.run() == mesos_pb2.DRIVER_STOPPED else 1
except:
status = 1
logger.exception('Mesos Scheduler Driver returned an exception')
#Perform a shut down and return any non-zero status
shutdown_status = self._shutdown()
status = status or shutdown_status
logger.info('Exiting...')
sys.exit(status)
def create_driver(framework_name,
scheduler,
system_paasta_config,
implicit_acks=False):
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = framework_name
framework.failover_timeout = 604800
framework.id.value = find_existing_id_if_exists_or_gen_new(framework.name)
framework.checkpoint = True
credential = mesos_pb2.Credential()
credential.principal = system_paasta_config.get_paasta_native_config(
)['principal']
credential.secret = system_paasta_config.get_paasta_native_config(
)['secret']
framework.principal = system_paasta_config.get_paasta_native_config(
)['principal']
driver = MesosSchedulerDriver(
scheduler, framework, '%s:%d' %
(mesos_tools.get_mesos_leader(), mesos_tools.MESOS_MASTER_PORT),
implicit_acks, credential)
return driver
def start_driver(self):
name = 'OpenCluster'
if self.options.name:
name = "%s-%s" % (name, self.options.name)
else:
name = "%s-%s" % (
name, datetime.datetime.now().strftime("%Y%m%d%H%M%S%f"))
if len(name) > 256:
name = name[:256] + '...'
framework = mesos_pb2.FrameworkInfo()
framework.user = getuser()
if framework.user == 'root':
raise Exception("OpenCluster is not allowed to run as 'root'")
framework.name = name
framework.hostname = socket.gethostname()
self.driver = MesosSchedulerDriver(self, framework, self.master)
self.driver.start()
logger.debug("Mesos Scheudler driver started")
self.shuttingdown = False
self.last_finish_time = time.time()
self.stopped = False
def __init__(self, scheduler, name, user='', master=os.getenv('MESOS_MASTER'),
implicit_acknowledge=1, *args, **kwargs):
framework = FrameworkInfo(name=name, user=user, *args, **kwargs)
scheduler = SchedulerProxy(scheduler)
self.driver = MesosSchedulerDriver(scheduler, encode(framework),
master, implicit_acknowledge)
def shutdown(signal, frame):
self.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.stop)
def main(master):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = 'MinimalExecutor'
executor.name = executor.executor_id.value
executor.command.value = os.path.abspath('./executor-minimal.py')
framework = mesos_pb2.FrameworkInfo()
framework.user = '' # the current user
framework.name = 'MinimalFramework'
framework.checkpoint = True
framework.principal = framework.name
implicitAcknowledgements = 1
driver = MesosSchedulerDriver(
MinimalScheduler(executor),
framework,
master,
implicitAcknowledgements
)
def signal_handler(signal, frame):
driver.stop()
def run_driver_thread():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
driver_thread = Thread(target=run_driver_thread, args=())
driver_thread.start()
print('Scheduler running, Ctrl-C to quit.')
signal.signal(signal.SIGINT, signal_handler)
while driver_thread.is_alive():
time.sleep(1)
sys.exit(0)
print "NASEL FRAMEWORK"
reuse_framework_id = framework['id']
#exit(1)
#exit(1)
except ZookeeperNoMaster as ex:
logging.error("Could not find any Mesos Master: {}".format(ex.message))
exit(1)
framework = mesos_pb2.FrameworkInfo()
framework.user = "******" # Have Mesos fill in the current user.
framework.name = "vizceral"
framework.failover_timeout=300
#print framework
if reuse_framework_id!="":
mesos_framework = mesos_pb2.FrameworkID()
mesos_framework.value=reuse_framework_id
framework.id.MergeFrom(mesos_framework)
logging.error("re-registring existing framework "+reuse_framework_id)
driver = MesosSchedulerDriver(
VizCeralScheduler(),
framework,
ZK_HOSTS
)
driver.run()
def main():
global shutdown
global accept_offers
global driver
cfg = get_configuration()
# Configure logging
setup_logging(cfg)
framework = mesos_framework(cfg)
#credentials = mesos_credentials(cfg)
mesos_scheduler = OurJobScheduler(cfg)
#driver = MesosSchedulerDriver(mesos_scheduler, framework,
# cfg.mesos.master, cfg.mesos.imp_ack,
# credentials)
driver = MesosSchedulerDriver(mesos_scheduler, framework, cfg.mesos.master,
cfg.mesos.imp_ack)
shutdown = Shutdown()
accept_offers = AcceptOffers()
# driver.run() blocks, so run it in a separate thread.
def run_driver_async():
status = 0 if driver.run() == MesosPb2.DRIVER_STOPPED else 1
if cfg.debug > 0:
logger.debug('Stopping Driver')
driver.stop()
logger.info('Terminating Framework')
sys.exit(status)
framework_thread = Thread(target=run_driver_async, args=())
framework_thread.start()
logger.info('Beginning Processing')
while framework_thread.is_alive():
# If a shutdown has been requested, suppress offers and wait for the
# framework thread to complete.
if shutdown.flag:
if cfg.debug > 0:
logger.debug('Suppressing Offers')
driver.suppressOffers()
while framework_thread.is_alive():
logger.debug('Child Thread Still Alive')
sleep(5)
break
# If the max number of jobs are already running, suppress offers and
# wait for some jobs to finish.
if (mesos_scheduler.tasks_launched == cfg.mesos.max_jobs):
driver.suppressOffers()
if cfg.debug > 0:
logger.debug('Suppressing Offers')
# Sleep until we have room for more tasks
while (not shutdown.flag
and mesos_scheduler.tasks_launched == cfg.mesos.max_jobs):
if cfg.debug > 0:
logger.debug('Waiting for more available tasks')
sleep(5)
# Sleep until more processing is requested
while not shutdown.flag and not mesos_scheduler.have_work():
if cfg.debug > 0:
logger.debug('Waiting for more work')
sleep(5)
if not shutdown.flag:
if cfg.debug > 0:
logger.debug('Reviving Offers')
driver.reviveOffers()
# If there's no new work to be done, suppress offers until we have
# more work
if not shutdown.flag and not mesos_scheduler.have_work():
driver.suppressOffers()
# Sleep until more processing is requested
while not shutdown.flag and not mesos_scheduler.have_work():
if cfg.debug > 0:
logger.debug('Waiting for more work')
sleep(5)
if not shutdown.flag:
if cfg.debug > 0:
logger.debug('Reviving Offers')
driver.reviveOffers()
# Sleep for a second, so that we are not flying through the loop
sleep(1)
logger.info('Terminated Processing')
def start_factory_mesos():
global pyroLoopCondition
parser = OptionParser(
usage="Usage: python factorymesos.py [options] <command>")
parser.allow_interspersed_args = False
parser.add_option("-s",
"--master",
type="string",
default="",
help="url of master (mesos://172.31.252.180:5050)")
parser.add_option("-f",
"--factory",
type="string",
default="",
help="host:port of master (172.31.252.180:6666)")
parser.add_option(
"-w",
"--warehouse_addr",
type="string",
default="",
help=
"kafka-172.31.252.182:9092|mysql-172.31.254.25:3306,db,username,password"
)
parser.add_option("-p",
"--task_per_node",
type="int",
default=0,
help="max number of tasks on one node (default: 0)")
parser.add_option("-I",
"--image",
type="string",
help="image name for Docker")
parser.add_option("-V",
"--volumes",
type="string",
help="volumes to mount into Docker")
parser.add_option("-r",
"--retry",
type="int",
default=0,
help="retry times when failed (default: 0)")
parser.add_option(
"-e",
"--config",
type="string",
default="/work/opencluster/config.ini",
help=
"absolute path of configuration file(default:/work/opencluster/config.ini)"
)
parser.add_option("-g",
"--group",
type="string",
default='',
help="which group to run (default: ''")
parser.add_option(
"-q",
"--quiet",
action="store_true",
help="be quiet",
)
parser.add_option(
"-v",
"--verbose",
action="store_true",
help="show more useful log",
)
(options, command) = parser.parse_args()
if not options:
parser.print_help()
sys.exit(2)
if options.config:
Conf.setConfigFile(options.config)
options.master = options.master or Conf.getMesosMaster()
options.warehouse_addr = options.warehouse_addr or Conf.getWareHouseAddr()
servers = options.factory or Conf.getFactoryServers()
servs = servers.split(",")
server = servs[0].split(":")
options.logLevel = (options.quiet and logging.ERROR
or options.verbose and logging.DEBUG or logging.INFO)
setLogger(Conf.getFactoryServiceName(), "MESOS", options.logLevel)
implicitAcknowledgements = 1
if os.getenv("MESOS_EXPLICIT_ACKNOWLEDGEMENTS"):
implicitAcknowledgements = 0
sched = FactoryMesos(options, command, implicitAcknowledgements)
driver = MesosSchedulerDriver(sched, sched.framework, options.master,
implicitAcknowledgements)
driver.start()
logger.debug("Mesos Scheudler driver started")
warehouse_addrs = options.warehouse_addr.split(",")
def fetchTasksFromMySQL():
global pyroLoopCondition
mysqlIpAndPort = warehouse_addrs[0].split(":")
last_data_time = time.time()
while pyroLoopCondition:
db = MySQLdb.connect(host=mysqlIpAndPort[0],
port=int(mysqlIpAndPort[1]),
db=warehouse_addrs[1],
user=warehouse_addrs[2],
passwd=warehouse_addrs[3])
try:
cur = db.cursor()
curUpt = db.cursor()
dataResults = cur.execute(
"select task_id,task_desc,task_start_time,status from t_task where status=0 order by priority asc limit 200"
)
results = cur.fetchmany(dataResults)
for r in results:
sched.append_task(cPickle.loads(r[1]))
curUpt.execute(
"update t_task set task_start_time=now(),status=1 where task_id='"
+ r[0] + "'")
if len(results) > 0:
db.commit()
last_data_time = time.time()
driver.reviveOffers()
if sched.tasks_total_len() > MAX_WAITING_TASK:
time.sleep(2)
if time.time() - last_data_time > MAX_EMPTY_TASK_PERIOD:
time.sleep(10)
if cur:
cur.close()
if curUpt:
curUpt.close()
finally:
db.close()
def fetchTasksFromKafka(priority):
global pyroLoopCondition
consumer = KafkaConsumer('OpenCluster%s' % priority,
bootstrap_servers=[options.warehouse_addr],
group_id="cnlab",
auto_commit_enable=True,
auto_commit_interval_ms=30 * 1000,
auto_offset_reset='smallest')
last_data_time = time.time()
while pyroLoopCondition:
for message in consumer.fetch_messages():
logger.error("%s:%s:%s: key=%s " %
(message.topic, message.partition, message.offset,
message.key))
sched.append_task(cPickle.loads(message.value))
consumer.task_done(message)
last_data_time = time.time()
if sched.tasks_len(priority) > MAX_WAITING_TASK:
time.sleep(2)
if time.time() - last_data_time > MAX_EMPTY_TASK_PERIOD:
time.sleep(10)
if len(warehouse_addrs) > 2:
spawn(fetchTasksFromMySQL)
else:
for i in range(1, sched.priority_size + 1):
spawn(fetchTasksFromKafka, i)
def handler(signm, frame):
logger.warning("got signal %d, exit now", signm)
sched.stop(3)
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGABRT, handler)
try:
while not sched.stopped:
time.sleep(0.5)
sched.check(driver)
now = time.time()
if now > sched.last_offer_time + 60 + random.randint(0, 5):
logger.warning("too long to get offer, reviving...")
sched.last_offer_time = now
driver.reviveOffers()
except KeyboardInterrupt:
logger.warning(
'stopped by KeyboardInterrupt. The Program is exiting gracefully! Please wait...'
)
sched.stop(4)
#terminate pyrothread
pyroLoopCondition = False
time.sleep(5)
driver.stop(False)
sys.exit(sched.status)
def run(api_url,
mesos_master,
user,
config_dir,
state_file,
changes_request_limit,
http_port,
stats=None):
scheduler = ChangesScheduler(state_file,
api=ChangesAPI(api_url),
stats=stats,
blacklist=FileBlacklist(
os.path.join(config_dir, 'blacklist')),
changes_request_limit=changes_request_limit)
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.command.value = os.path.abspath("./executor.py")
executor.name = "Changes Executor"
executor.source = "changes"
framework = mesos_pb2.FrameworkInfo()
framework.user = user
framework.name = "Changes Scheduler"
framework.principal = "changes"
# Give the scheduler 1 week to restart before mesos cancels the tasks.
# this is the setting recommended by the docs.
framework.failover_timeout = 3600 * 24 * 7
if scheduler.framework_id:
framework.id.value = scheduler.framework_id
executor.framework_id.value = scheduler.framework_id
driver = MesosSchedulerDriver(scheduler, framework, mesos_master)
stopped = threading.Event()
def handle_interrupt(signal, frame):
stopped.set()
logging.info("Received interrupt, shutting down")
logging.warning(
"Not saving state. Will wait for running tasks to finish.")
scheduler.shuttingDown.set()
while scheduler.activeTasks > 0:
logging.info("Waiting for %d tasks to finish running",
scheduler.activeTasks)
sleep(5)
driver.stop()
def handle_sigterm(signal, frame):
# TODO: Avoid save_state race conditions by having handle_sigterm()
# only set shuttingDown, then do the actual save-state and driver.stop()
# in the main thread after all other threads are join()ed.
# Also, stopped doesn't appear to be used.
stopped.set()
logging.info("Received sigterm, shutting down")
scheduler.shuttingDown.set()
if scheduler.state_file:
try:
scheduler.save_state()
logging.info("Successfully saved state to %s.", state_file)
except Exception:
logging.exception("Failed to save state")
driver.stop()
return
# With `failover` set to true, we do not tell Mesos to stop the existing tasks
# started by this framework. Instead, the tasks will run for
# `fail_timeout` more seconds set above or we start a scheduler with
# the same framework id.
driver.stop(True)
else:
logging.warning(
"State file location not set. Not saving state. Existing builds will be cancelled."
)
driver.stop()
signal.signal(signal.SIGINT, handle_interrupt)
signal.signal(signal.SIGTERM, handle_sigterm)
driver.start()
logging.info("Driver started")
app = Flask("Changes Mesos Scheduler")
app.add_url_rule('/api/state_json', 'state_json',
json_handler(scheduler.state_json))
http_thread = threading.Thread(target=app.run, kwargs={'port': http_port})
http_thread.start()
scheduler.poll_changes_until_shutdown(driver, 5)
status = 0
if driver.join() == mesos_pb2.DRIVER_STOPPED:
logging.info("Driver stopped cleanly.")
else:
# Ensure that the driver process terminates.
status = 1
logging.info("Stopping driver forcibly.")
driver.stop()
logging.info("Stopping HTTP server.")
http_thread.terminate()
http_thread.join()
logging.info("Clean shutdown complete. Exiting status %d.", status)
sys.exit(status)
task = mesos_pb2.TaskInfo()
id = uuid.uuid4()
task.task_id.value = str(id)
task.slave_id.value = offer.slave_id.value
task.name = "task {}".format(str(id))
cpus = task.resources.add()
cpus.name = "cpus"
cpus.type = mesos_pb2.Value.SCALAR
cpus.scalar.value = 0.1
mem = task.resources.add()
mem.name = "mem"
mem.type = mesos_pb2.Value.SCALAR
mem.scalar.value = 32
return task
if __name__ == '__main__':
log("XXX framework started")
framework = mesos_pb2.FrameworkInfo()
framework.user = "******"
framework.name = "nixos-test-framework"
driver = MesosSchedulerDriver(
NixosTestScheduler(),
framework,
sys.argv[1] + ":5050"
)
driver.run()
task.command.value = "python install_nuage_cni.py %s %s" % (
sys.argv[2], sys.argv[3])
time.sleep(2)
logging.info(
"Launching task {task} "
"using offer {offer}.".format(task=task.task_id.value,
offer=offer.id.value))
tasks = [task]
driver.launchTasks(offer.id, tasks)
def statusUpdate(self, driver, update):
print "Task %s is in state %d" % (update.task_id.value, update.state)
if update.state == mesos_pb2.TASK_FINISHED:
print "Task successfully executed on agent node"
self.serviced_agents += 1
if self.serviced_agents == len(agent_list):
driver.stop()
if __name__ == '__main__':
# Create a Mesos framework for CNI installation
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "install-cni"
driver = MesosSchedulerDriver(
InstallCNIScheduler(),
framework,
sys.argv[1] + ":5050" # assumes running on the master
)
driver.run()
uri_proto.extract = False
vilfredo = VilfredoMesosScheduler(paretoExecutor, sys.argv[2])
# Use credentials for authentication if they are provided.
credential = None
if len(sys.argv) > 3:
credentialsPath = sys.argv[3]
with open(credentialsPath) as f:
words = f.readline().split()
credential = mesos_pb2.Credential()
credential.principal = words[0]
credential.secret = words[1]
if credential is None:
driver = MesosSchedulerDriver(vilfredo, framework, sys.argv[1])
else:
driver = MesosSchedulerDriver(vilfredo, framework, sys.argv[1],
credential)
# driver.run() blocks; we run it in a separate thread.
def run_driver_async():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
framework_thread = threading.Thread(target=run_driver_async)
framework_thread.start()
print "(Listening for Ctrl-C)"
signal.signal(signal.SIGINT, hard_shutdown)
hello_executor.name = "Hello"
hello_executor.command.value = "python hello_executor.py"
uri_proto = hello_executor.command.uris.add()
uri_proto.value = "http://kit-mesos-master:9000/hello_executor.py"
uri_proto.extract = False
goodbye_executor = mesos_pb2.ExecutorInfo()
goodbye_executor.executor_id.value = "goodbye-executor"
goodbye_executor.name = "GoodBye"
goodbye_executor.command.value = "python goodbye_executor.py"
uri_proto = goodbye_executor.command.uris.add()
uri_proto.value = "http://kit-mesos-master:9000/goodbye_executor.py"
uri_proto.extract = False
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
helloWorldScheduler = HelloWorldScheduler(hello_executor, goodbye_executor)
driver = MesosSchedulerDriver(helloWorldScheduler, framework,
"kit-mesos-master:5050")
driver.start()
logging.info("Listening for Ctrl-C")
signal.signal(signal.SIGINT, graceful_shutdown)
while True:
time.sleep(5)
sys.exit(0)
goodbye_executor.name = "GoodBye"
goodbye_executor.command.value = "python goodbye_executor.py"
uri_proto = goodbye_executor.command.uris.add()
uri_proto.value = "http://kit-mesos-master:9000/goodbye_executor.py"
uri_proto.extract = False
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
httpd = SocketServer.TCPServer(("", 9000),
SimpleHTTPServer.SimpleHTTPRequestHandler)
def create_web_server():
print "serving at port", 9000
httpd.serve_forever()
web_thread = threading.Thread(target=create_web_server)
web_thread.start()
driver = MesosSchedulerDriver(
HelloWorldScheduler(hello_executor, goodbye_executor), framework,
"zk://localhost:2181/mesos")
driver.start()
logging.info("Listening for Ctrl-C")
signal.signal(signal.SIGINT, shutdown)
while True:
time.sleep(5)
sys.exit(0)
logging.info("task {task} status is {status}".format(
task=update.task_id.value,
status=mesos_pb2.TaskState.Name(update.state)))
if update.state == mesos_pb2.TASK_FINISHED:
self.finished_task += 1
if self.finished_task == len(self.numbers):
logging.info("all task has finished")
driver.stop()
if __name__ == '__main__':
# make us a framework
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.command.value = "python {}".format(
os.path.abspath("./Executor.py"))
executor.name = "Test Executor(Python)"
executor.source = "python test"
driver = MesosSchedulerDriver(
MyScheduler(executor),
framework,
"192.168.12.179:5050" # assumes running on the master
)
driver.run()
def hard_shutdown(signal, frame):
print "Shutting down..."
driver.stop()
if __name__ == "__main__":
if len(sys.argv) != 3:
print "Usage: %s master command" % sys.argv[0]
sys.exit(1)
framework = mesos_pb2.FrameworkInfo()
framework.user = ""
framework.name = "Lanceur (Simple Launcher Framework in python)"
driver = MesosSchedulerDriver(LaunchScheduler(sys.argv[2]), framework,
sys.argv[1])
print "Starting Mesos driver"
# driver.run() blocks; we run it in a separate thread
def run_driver_async():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
framework_thread = threading.Thread(target=run_driver_async)
framework_thread.start()
print "(Listening for Ctrl-C)"
signal.signal(signal.SIGINT, hard_shutdown)
while framework_thread.is_alive():
uri_proto.extract = False
renderExecutor.name = "Renderer"
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "RENDLER"
try:
maxRenderTasks = int(sys.argv[3])
except:
maxRenderTasks = 0
rendler = RenderingCrawler(sys.argv[1], maxRenderTasks, crawlExecutor,
renderExecutor)
driver = MesosSchedulerDriver(rendler, framework, sys.argv[2])
# driver.run() blocks; we run it in a separate thread
def run_driver_async():
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
driver.stop()
sys.exit(status)
framework_thread = Thread(target=run_driver_async, args=())
framework_thread.start()
print "(Listening for Ctrl-C)"
signal.signal(signal.SIGINT, graceful_shutdown)
while framework_thread.is_alive():
time.sleep(1)
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
helloWorldScheduler = HelloWorldScheduler(hello_executor, goodbye_executor)
httpd = SocketServer.TCPServer(
("", 9000),
SimpleHTTPServer.SimpleHTTPRequestHandler)
def create_web_server():
print "serving at port", 9000
httpd.serve_forever()
thread = threading.Thread(target=create_web_server)
thread.start()
driver = MesosSchedulerDriver(
helloWorldScheduler,
framework,
"zk://localhost:2181/mesos"
)
driver.start()
logging.info("Listening for Ctrl-C")
signal.signal(signal.SIGINT, graceful_shutdown)
while True:
time.sleep(5)
sys.exit(0)
"using offer %s.",
task.task_id.value,
offer.id.value)
driver.launchTasks(offer.id, [task])
def statusUpdate(self, driver, update):
'''
when a task is started, over,
killed or lost (slave crash, ....), this method
will be triggered with a status message.
'''
logging.info("Task %s is in state %s" %
(update.task_id.value,
mesos_pb2.TaskState.Name(update.state)))
if __name__ == '__main__':
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
driver = MesosSchedulerDriver(
HelloWorldScheduler(),
framework,
"zk://localhost:2181/mesos" # assumes running on the master
)
driver.start()
logging.info("Listening for Ctrl-C")
signal.signal(signal.SIGINT, shutdown)
while True:
time.sleep(5)
sys.exit(0)
def registered(self, driver, framework_id, master_info):
logging.info("Registered with framework id: {}".format(framework_id))
def resourceOffers(self, driver, offers):
logging.info("Recieved resource offers: {}".format(
[o.id.value for o in offers]))
# whenever we get an offer, we accept it and use it to launch a task that
# just echos hello world to stdout
for offer in offers:
task = new_task(offer)
task.command.value = "echo hello world"
time.sleep(2)
logging.info("Launching task {task} "
"using offer {offer}.".format(task=task.task_id.value,
offer=offer.id.value))
tasks = [task]
driver.launchTasks(offer.id, tasks)
if __name__ == '__main__':
# make us a framework
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "hello-world"
driver = MesosSchedulerDriver(
HelloWorldScheduler(),
framework,
"zk://34.215.39.254/mesos" # assumes running on the master
)
driver.run()
