def main():
# Create executor stub
thermos_gc_executor = ThermosGCExecutor(
checkpoint_root=TaskPath.DEFAULT_CHECKPOINT_ROOT)
thermos_gc_executor.start()
# Start metrics collection
metric_writer = DiskMetricWriter(thermos_gc_executor.metrics,
ExecutorDetector.VARS_PATH)
metric_writer.start()
# Create driver stub
driver = mesos.MesosExecutorDriver(thermos_gc_executor)
# Start GC executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main():
runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# Create executor stub
thermos_executor = ThermosExecutor(
runner_provider=runner_provider,
status_providers=(HealthCheckerProvider(), ),
)
# Create driver stub
driver = mesos.MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def __init__(self):
mesos.Executor.__init__(self)
self.tid = -1
self.nested_driver = -1
def launchTask(self, driver, task):
self.tid = task.taskId
master, (todo, duration) = pickle.loads(task.arg)
scheduler = NestedScheduler(todo, duration, self)
print "Running here:" + master
self.nested_driver = mesos.MesosSchedulerDriver(scheduler, master)
self.nested_driver.start()
def killTask(self, driver, tid):
if (tid != self.tid):
print "Expecting different task id ... killing anyway!"
if self.nested_driver != -1:
self.nested_driver.stop()
self.nested_driver.join()
driver.sendStatusUpdate(mesos.TaskStatus(tid, mesos.TASK_FINISHED, ""))
def shutdown(self, driver):
self.killTask(self.tid)
def error(self, driver, code, message):
print "Error: %s" % message
if __name__ == "__main__":
mesos.MesosExecutorDriver(ScalingExecutor()).run()
def killTask(self, driver, task_id):
logging.debug( "Killing task %s" % task_id.value )
update = mesos_pb2.TaskStatus()
update.task_id.value = task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = json.dumps( { 'hostname' : socket.gethostname(), 'task_id' : task_id.value } )
driver.sendStatusUpdate(update)
def frameworkMessage(self, driver, message):
# Send it back to the scheduler.
if message == "diskspace":
pass
def shutdown(self, driver):
logging.debug( "shutdown" )
#cleanup()
def error(self, driver, code, message):
print "Error: %s" % message
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("-w", "--workdir", default="/tmp")
parser.add_argument("--docker", default=None)
parser.add_argument("--storage-dir", default="/tmp/nebula-store")
args = parser.parse_args()
logging.info( "Starting Workflow Watcher" )
executor = NebulaExecutor(args)
mesos.MesosExecutorDriver(executor).run()
driver.sendFrameworkMessage(message)
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update"
return
thread = threading.Thread(target=run_task)
thread.start()
def killTask(self, driver, taskId):
pass
def frameworkMessage(self, driver, message):
pass
def shutdown(self, driver):
pass
def error(self, error, message):
pass
if __name__ == "__main__":
print "Starting Launching Executor (LE)"
driver = mesos.MesosExecutorDriver(CrawlExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def __init__(self):
mesos.Executor.__init__(self)
self.tid = -1
def init(self, driver, args):
self.fid = args.frameworkId
def launchTask(self, driver, task):
self.tid = task.taskId
duration = pickle.loads(task.arg)
print "(%s:%d) Sleeping for %s seconds." % (self.fid, self.tid,
duration)
# TODO(benh): Don't sleep, this blocks the event loop!
time.sleep(duration)
status = mesos.TaskStatus(self.tid, mesos.TASK_FINISHED, "")
driver.sendStatusUpdate(status)
time.sleep(1)
def killTask(self, driver, tid):
if (self.tid != tid):
print "Expecting different task id ... killing anyway!"
status = mesos.TaskStatus(tid, mesos.TASK_FINISHED, "")
driver.sendStatusUpdate(status)
def error(self, driver, code, message):
print "Error: %s" % message
if __name__ == "__main__":
mesos.MesosExecutorDriver(NestedExecutor()).run()
def run_task():
print "Running task %s" % task.task_id.value
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_RUNNING
update.data = 'data with a \0 byte'
driver.sendStatusUpdate(update)
# This is where one would perform the requested task.
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
update.data = 'data with a \0 byte'
driver.sendStatusUpdate(update)
print "Sent status update"
thread = threading.Thread(target=run_task)
thread.start()
def frameworkMessage(self, driver, message):
# Send it back to the scheduler.
driver.sendFrameworkMessage(message)
if __name__ == "__main__":
print "Starting executor"
driver = mesos.MesosExecutorDriver(MyExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def run():
executor = MyExecutor()
driver = mesos.MesosExecutorDriver(executor)
driver.run()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update for task %s" % task.task_id.value
return
thread = threading.Thread(target=run_task)
thread.start()
def killTask(self, driver, taskId):
pass
def frameworkMessage(self, driver, message):
pass
def shutdown(self, driver):
pass
def error(self, error, message):
pass
if __name__ == "__main__":
print "Starting Render Executor (RE)"
local = False
if len(sys.argv) == 2 and sys.argv[1] == "--local":
local = True
driver = mesos.MesosExecutorDriver(RenderExecutor(local))
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
#!/usr/bin/env python
import mesos
import sys
import time
class MyExecutor(mesos.Executor):
def __init__(self):
mesos.Executor.__init__(self)
def launchTask(self, driver, task):
print "Running task %d" % task.taskId
time.sleep(1)
print "Sending the update..."
update = mesos.TaskStatus(task.taskId, mesos.TASK_FINISHED, "")
driver.sendStatusUpdate(update)
print "Sent the update"
def error(self, driver, code, message):
print "Error: %s" % message
if __name__ == "__main__":
print "Starting executor"
mesos.MesosExecutorDriver(MyExecutor()).run()
def run_executor():
"""run the executor until it is stopped externally by the framework"""
driver = mesos.MesosExecutorDriver(MesosExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def cli():
schedulers = {
"sleep": SleepScheduler,
"pg": PGScheduler,
"executor": ExecutorScheduler
}
p = argparse.ArgumentParser(prog="deimos-test.py")
p.add_argument("--master",
default="localhost:5050",
help="Mesos master URL")
p.add_argument("--test",
choices=schedulers.keys(),
default="sleep",
help="Test scheduler to use")
p.add_argument("--executor",
action="store_true",
default=False,
help="Runs the executor instead of a test scheduler")
p.add_argument("--test.container", help="Image URL to use (for any test)")
p.add_argument("--test.uris",
action="append",
help="Pass any number of times to add URIs (for any test)")
p.add_argument("--test.trials",
type=int,
help="Number of tasks to run (for any test)")
p.add_argument("--test.sleep",
type=int,
help="Seconds to sleep (for sleep test)")
p.add_argument("--test.command", help="Command to use (for executor test)")
parsed = p.parse_args()
if parsed.executor:
log.info("Mesos executor mode was chosen")
driver = mesos.MesosExecutorDriver(ExecutorSchedulerExecutor())
code = driver.run()
log.info(mesos_pb2.Status.Name(code))
driver.stop()
if code != mesos_pb2.DRIVER_STOPPED:
log.error("Driver died in an anomalous state")
os._exit(2)
os._exit(0)
pairs = [(k.split("test.")[1:], v) for k, v in vars(parsed).items()]
constructor_args = dict((k[0], v) for k, v in pairs if len(k) == 1 and v)
scheduler_class = schedulers[parsed.test]
scheduler = scheduler_class(**constructor_args)
args = ", ".join("%s=%r" % (k, v) for k, v in constructor_args.items())
log.info("Testing: %s(%s)" % (scheduler_class.__name__, args))
framework = mesos_pb2.FrameworkInfo()
framework.name = "deimos-test"
framework.user = ""
driver = mesos.MesosSchedulerDriver(scheduler, framework, parsed.master)
code = driver.run()
log.info(mesos_pb2.Status.Name(code))
driver.stop()
################ 2 => driver problem 1 => tests failed 0 => tests passed
if code != mesos_pb2.DRIVER_STOPPED:
log.error("Driver died in an anomalous state")
log.info("Aborted: %s(%s)" % (scheduler_class.__name__, args))
os._exit(2)
if any(_ in Scheduler.failed for _ in scheduler.statuses.values()):
log.error("Test run failed -- not all tasks made it")
log.info("Failure: %s(%s)" % (scheduler_class.__name__, args))
os._exit(1)
log.info("Success: %s(%s)" % (scheduler_class.__name__, args))
os._exit(0)
target_path = os.path.join(follow_dir, name + TARGET_SUFFIX)
handle = open(target_path, "w")
handle.write(pickle.dumps(target))
handle.close()
fail = False
except Exception, e:
log(str(e))
log("Sending status update...")
update = mesos_pb2.TaskStatus()
update.task_id.value = task_path
if fail:
update.state = mesos_pb2.TASK_FAILED
else:
update.state = mesos_pb2.TASK_FINISHED
update.data = str(task_path)
driver.sendStatusUpdate(update)
log("Sent status update")
thread = threading.Thread(target=run_task)
thread.start()
def frameworkMessage(self, driver, message):
# Send it back to the scheduler.
driver.sendFrameworkMessage(message)
if __name__ == "__main__":
print "Starting executor"
driver = mesos.MesosExecutorDriver(MyExecutor(sys.argv[1]))
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
