def getExecutorInfo(self): """ Build an executor request structure """ uri_value = "%s/resources/nebula_executor.py" % (self.master_url) logging.info("in getExecutorInfo, setting execPath = " + uri_value) executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = "nebula_worker" uri = executor.command.uris.add() uri.value = uri_value uri.executable = True cmd = "./nebula_executor.py -w %s" % (self.config.workdir) if self.config.docker is not None: cmd += " --docker %s" % (self.config.docker) executor.command.value = cmd env_path = executor.command.environment.variables.add() env_path.name = "PATH" env_path.value = os.environ['PATH'] executor.name = "nebula_worker" executor.source = "nebula_farm" return executor
def getExecutorInfo(self): frameworkDir = os.path.abspath(os.path.dirname(sys.argv[0])) executorPath = os.path.join(frameworkDir, "executor") execInfo = mesos_pb2.ExecutorInfo() execInfo.executor_id.value = "default" execInfo.uri = executorPath return execInfo
def getExecutorInfo(self, driver): execPath = os.path.join(os.getcwd(), "startmpd.sh") initArg = ip + ":" + port execInfo = mesos_pb2.ExecutorInfo() execInfo.executor_id.value = "default" execInfo.uri = execPath execInfo.data = initArg return execInfo
def task_with_executor(tid, sid, eid, *args): executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = eid executor.name = eid executor.command.MergeFrom(command(*args)) task = task_base(tid, sid) task.executor.MergeFrom(executor) return task
def getExecutorInfo(self): frameworkDir = os.path.abspath(os.path.dirname(sys.argv[0])) executorPath = os.path.join(frameworkDir, "executor.py") execInfo = mesos_pb2.ExecutorInfo() execInfo.executor_id.value = "default" execInfo.command.value = executorPath if hasattr(execInfo, 'framework_id'): execInfo.framework_id.value = str(self.framework_id) return execInfo
def start_framework(master_uri, exe_path, n_workers, uow_name, prefix, cpu_alloc, mem_alloc): # initialize an executor executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = uuid1().hex executor.command.value = exe_path executor.name = "Exelixi Executor" executor.source = "per-job build" ## NB: TODO download tarball/container from HDFS #uri = executor.command.uris.add() #uri.executable = false #uri.value = "hdfs://namenode/exelixi/exelixi.tgz" # initialize the framework framework = mesos_pb2.FrameworkInfo() framework.user = "" # have Mesos fill in the current user framework.name = "Exelixi Framework" if os.getenv("MESOS_CHECKPOINT"): logging.debug( "Mesos Scheduler: enabling checkpoint for the framework") framework.checkpoint = True # create a scheduler and capture the command line options sched = MesosScheduler(executor, exe_path, n_workers, uow_name, prefix, cpu_alloc, mem_alloc) # initialize a driver if os.getenv("MESOS_AUTHENTICATE"): logging.debug( "Mesos Scheduler: enabling authentication for the framework") if not os.getenv("DEFAULT_PRINCIPAL"): logging.critical( "Mesos Scheduler: expecting authentication principal in the environment" ) sys.exit(1) if not os.getenv("DEFAULT_SECRET"): logging.critical( "Mesos Scheduler: expecting authentication secret in the environment" ) sys.exit(1) credential = mesos_pb2.Credential() credential.principal = os.getenv("DEFAULT_PRINCIPAL") credential.secret = os.getenv("DEFAULT_SECRET") driver = mesos.MesosSchedulerDriver(sched, framework, master_uri, credential) else: driver = mesos.MesosSchedulerDriver(sched, framework, master_uri) return driver
def run(self): self.queue.branchUpdate("/") executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = "default" executor.command.value = "PYTHONPATH=%s %s " % ( os.environ['PYTHONPATH'], sys.executable) + os.path.abspath( os.path.join(os.path.dirname(__file__), "./jobTreeExec.py %s" % (self.common_dir))) executor.name = "JobTreeExec" executor.source = "JobTree" framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = "JobTree" # TODO(vinod): Make checkpointing the default when it is default # on the slave. if os.getenv("MESOS_CHECKPOINT"): logging.info("Enabling checkpoint for the framework") framework.checkpoint = True if os.getenv("MESOS_AUTHENTICATE"): logging.info("Enabling authentication for the framework") if not os.getenv("DEFAULT_PRINCIPAL"): logging.error( "Expecting authentication principal in the environment") return if not os.getenv("DEFAULT_SECRET"): logging.error( "Expecting authentication secret in the environment") return credential = mesos_pb2.Credential() credential.principal = os.getenv("DEFAULT_PRINCIPAL") credential.secret = os.getenv("DEFAULT_SECRET") driver = mesos.MesosSchedulerDriver( JobTreeScheduler(executor, self.queue), framework, self.mesos_url, credential) else: logging.info("Contacting Mesos: %s" % self.mesos_url) driver = mesos.MesosSchedulerDriver( JobTreeScheduler(executor, self.queue), framework, self.mesos_url) status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 logging.info("Status: %s" % status) # Ensure that the driver process terminates. driver.stop()
def getExecutorInfo(self): info = mesos_pb2.ExecutorInfo() if self.use_self_as_exec: info.command.value = os.path.abspath(sys.argv[0]) info.executor_id.value = sys.argv[0] else: dir = os.path.dirname(__file__) info.command.value = os.path.abspath(os.path.join(dir, 'executor%d%d.py' % sys.version_info[:2])) info.executor_id.value = "default" mem = info.resources.add() mem.name = 'mem' mem.type = 0 #mesos_pb2.Value.SCALAR mem.scalar.value = EXECUTOR_MEMORY info.data = marshal.dumps((os.path.realpath(sys.argv[0]), os.getcwd(), sys.path, self.task_per_node, self.out_logger, self.err_logger, self.logLevel, env.environ)) return info
print "Received message:", repr(str(message)) if self.messagesReceived == TOTAL_TASKS: if self.messagesReceived != self.messagesSent: print "Sent", self.messagesSent, print "but received", self.messagesReceived sys.exit(1) print "All tasks done, and all messages received, exiting" driver.stop() if __name__ == "__main__": if len(sys.argv) != 2: print "Usage: %s master" % sys.argv[0] sys.exit(1) executor = mesos_pb2.ExecutorInfo() executor.executor_id.value = "default" executor.command.value = os.path.abspath("./test-executor") executor.name = "Test Executor (Python)" executor.source = "python_test" framework = mesos_pb2.FrameworkInfo() framework.user = "" # Have Mesos fill in the current user. framework.name = "Test Framework (Python)" driver = mesos.MesosSchedulerDriver(TestScheduler(executor), framework, sys.argv[1]) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
""" print("Error from Mesos: %s " % message, file=sys.stderr) # # Execution entry point: # if __name__ == "__main__": if len(sys.argv) < 3 or len(sys.argv) > 4: print "Usage: %s seedUrl mesosMasterUrl [--local]" % sys.argv[0] sys.exit(1) localMode = len(sys.argv) == 4 and sys.argv[3] == "--local" rendlerArtifact = "http://downloads.mesosphere.io/demo/rendler.tgz" crawlExecutor = mesos_pb2.ExecutorInfo() crawlExecutor.executor_id.value = "crawl-executor" crawlExecutor.command.value = "python crawl_executor.py" crawlExecutor.command.uris.add().value = rendlerArtifact crawlExecutor.name = "Crawler" crawlExecutor.source = "rendering-crawler" renderExecutor = mesos_pb2.ExecutorInfo() renderExecutor.executor_id.value = "render-executor" renderExecutor.command.value = "python render_executor.py" if localMode: renderExecutor.command.value += " --local" renderExecutor.command.uris.add().value = rendlerArtifact renderExecutor.name = "Renderer"