def getExecutorInfo(self):
"""
Build an executor request structure
"""
uri_value = "%s/resources/nebula_executor.py" % (self.master_url)
logging.info("in getExecutorInfo, setting execPath = " + uri_value)
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "nebula_worker"
uri = executor.command.uris.add()
uri.value = uri_value
uri.executable = True
cmd = "./nebula_executor.py -w %s" % (self.config.workdir)
if self.config.docker is not None:
cmd += " --docker %s" % (self.config.docker)
executor.command.value = cmd
env_path = executor.command.environment.variables.add()
env_path.name = "PATH"
env_path.value = os.environ['PATH']
executor.name = "nebula_worker"
executor.source = "nebula_farm"
return executor
def getExecutorInfo(self):
frameworkDir = os.path.abspath(os.path.dirname(sys.argv[0]))
executorPath = os.path.join(frameworkDir, "executor")
execInfo = mesos_pb2.ExecutorInfo()
execInfo.executor_id.value = "default"
execInfo.uri = executorPath
return execInfo
def getExecutorInfo(self, driver):
execPath = os.path.join(os.getcwd(), "startmpd.sh")
initArg = ip + ":" + port
execInfo = mesos_pb2.ExecutorInfo()
execInfo.executor_id.value = "default"
execInfo.uri = execPath
execInfo.data = initArg
return execInfo
def task_with_executor(tid, sid, eid, *args):
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = eid
executor.name = eid
executor.command.MergeFrom(command(*args))
task = task_base(tid, sid)
task.executor.MergeFrom(executor)
return task
def getExecutorInfo(self):
frameworkDir = os.path.abspath(os.path.dirname(sys.argv[0]))
executorPath = os.path.join(frameworkDir, "executor.py")
execInfo = mesos_pb2.ExecutorInfo()
execInfo.executor_id.value = "default"
execInfo.command.value = executorPath
if hasattr(execInfo, 'framework_id'):
execInfo.framework_id.value = str(self.framework_id)
return execInfo
def start_framework(master_uri, exe_path, n_workers, uow_name, prefix,
cpu_alloc, mem_alloc):
# initialize an executor
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = uuid1().hex
executor.command.value = exe_path
executor.name = "Exelixi Executor"
executor.source = "per-job build"
## NB: TODO download tarball/container from HDFS
#uri = executor.command.uris.add()
#uri.executable = false
#uri.value = "hdfs://namenode/exelixi/exelixi.tgz"
# initialize the framework
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # have Mesos fill in the current user
framework.name = "Exelixi Framework"
if os.getenv("MESOS_CHECKPOINT"):
logging.debug(
"Mesos Scheduler: enabling checkpoint for the framework")
framework.checkpoint = True
# create a scheduler and capture the command line options
sched = MesosScheduler(executor, exe_path, n_workers, uow_name, prefix,
cpu_alloc, mem_alloc)
# initialize a driver
if os.getenv("MESOS_AUTHENTICATE"):
logging.debug(
"Mesos Scheduler: enabling authentication for the framework")
if not os.getenv("DEFAULT_PRINCIPAL"):
logging.critical(
"Mesos Scheduler: expecting authentication principal in the environment"
)
sys.exit(1)
if not os.getenv("DEFAULT_SECRET"):
logging.critical(
"Mesos Scheduler: expecting authentication secret in the environment"
)
sys.exit(1)
credential = mesos_pb2.Credential()
credential.principal = os.getenv("DEFAULT_PRINCIPAL")
credential.secret = os.getenv("DEFAULT_SECRET")
driver = mesos.MesosSchedulerDriver(sched, framework, master_uri,
credential)
else:
driver = mesos.MesosSchedulerDriver(sched, framework, master_uri)
return driver
def run(self):
self.queue.branchUpdate("/")
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.command.value = "PYTHONPATH=%s %s " % (
os.environ['PYTHONPATH'], sys.executable) + os.path.abspath(
os.path.join(os.path.dirname(__file__), "./jobTreeExec.py %s" %
(self.common_dir)))
executor.name = "JobTreeExec"
executor.source = "JobTree"
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "JobTree"
# TODO(vinod): Make checkpointing the default when it is default
# on the slave.
if os.getenv("MESOS_CHECKPOINT"):
logging.info("Enabling checkpoint for the framework")
framework.checkpoint = True
if os.getenv("MESOS_AUTHENTICATE"):
logging.info("Enabling authentication for the framework")
if not os.getenv("DEFAULT_PRINCIPAL"):
logging.error(
"Expecting authentication principal in the environment")
return
if not os.getenv("DEFAULT_SECRET"):
logging.error(
"Expecting authentication secret in the environment")
return
credential = mesos_pb2.Credential()
credential.principal = os.getenv("DEFAULT_PRINCIPAL")
credential.secret = os.getenv("DEFAULT_SECRET")
driver = mesos.MesosSchedulerDriver(
JobTreeScheduler(executor, self.queue), framework,
self.mesos_url, credential)
else:
logging.info("Contacting Mesos: %s" % self.mesos_url)
driver = mesos.MesosSchedulerDriver(
JobTreeScheduler(executor, self.queue), framework,
self.mesos_url)
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
logging.info("Status: %s" % status)
# Ensure that the driver process terminates.
driver.stop()
def getExecutorInfo(self):
info = mesos_pb2.ExecutorInfo()
if self.use_self_as_exec:
info.command.value = os.path.abspath(sys.argv[0])
info.executor_id.value = sys.argv[0]
else:
dir = os.path.dirname(__file__)
info.command.value = os.path.abspath(os.path.join(dir, 'executor%d%d.py' % sys.version_info[:2]))
info.executor_id.value = "default"
mem = info.resources.add()
mem.name = 'mem'
mem.type = 0 #mesos_pb2.Value.SCALAR
mem.scalar.value = EXECUTOR_MEMORY
info.data = marshal.dumps((os.path.realpath(sys.argv[0]), os.getcwd(), sys.path, self.task_per_node,
self.out_logger, self.err_logger, self.logLevel, env.environ))
return info
print "Received message:", repr(str(message))
if self.messagesReceived == TOTAL_TASKS:
if self.messagesReceived != self.messagesSent:
print "Sent", self.messagesSent,
print "but received", self.messagesReceived
sys.exit(1)
print "All tasks done, and all messages received, exiting"
driver.stop()
if __name__ == "__main__":
if len(sys.argv) != 2:
print "Usage: %s master" % sys.argv[0]
sys.exit(1)
executor = mesos_pb2.ExecutorInfo()
executor.executor_id.value = "default"
executor.command.value = os.path.abspath("./test-executor")
executor.name = "Test Executor (Python)"
executor.source = "python_test"
framework = mesos_pb2.FrameworkInfo()
framework.user = "" # Have Mesos fill in the current user.
framework.name = "Test Framework (Python)"
driver = mesos.MesosSchedulerDriver(TestScheduler(executor), framework,
sys.argv[1])
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
"""
print("Error from Mesos: %s " % message, file=sys.stderr)
#
# Execution entry point:
#
if __name__ == "__main__":
if len(sys.argv) < 3 or len(sys.argv) > 4:
print "Usage: %s seedUrl mesosMasterUrl [--local]" % sys.argv[0]
sys.exit(1)
localMode = len(sys.argv) == 4 and sys.argv[3] == "--local"
rendlerArtifact = "http://downloads.mesosphere.io/demo/rendler.tgz"
crawlExecutor = mesos_pb2.ExecutorInfo()
crawlExecutor.executor_id.value = "crawl-executor"
crawlExecutor.command.value = "python crawl_executor.py"
crawlExecutor.command.uris.add().value = rendlerArtifact
crawlExecutor.name = "Crawler"
crawlExecutor.source = "rendering-crawler"
renderExecutor = mesos_pb2.ExecutorInfo()
renderExecutor.executor_id.value = "render-executor"
renderExecutor.command.value = "python render_executor.py"
if localMode:
renderExecutor.command.value += " --local"
renderExecutor.command.uris.add().value = rendlerArtifact
renderExecutor.name = "Renderer"
