class Running(object):
def __init__(self, executor):
executor = ExecutorProxy(executor)
self.driver = MesosExecutorDriver(executor)
def shutdown(signal, frame):
self.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.stop)
def run(self):
return self.driver.run()
def start(self):
status = self.driver.start()
assert status == mesos_pb2.DRIVER_RUNNING
return status
def stop(self):
return self.driver.stop()
def join(self):
return self.driver.join()
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.stop()
self.join()
if exc_type:
raise exc_type, exc_value, traceback
def main(args, options):
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# status providers:
status_providers = [HealthCheckerProvider()]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error('Must specify --announcer-ensemble if the announcer is enabled.')
status_providers.append(DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path))
# Create executor stub
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def __init__(self, executor):
executor = ExecutorProxy(executor)
self.driver = MesosExecutorDriver(executor)
def shutdown(signal, frame):
self.driver.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.driver.stop)
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=options.checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error('Must specify --announcer-ensemble if the announcer is enabled.')
status_providers.append(DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.abspath(CWD)
)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user)
)
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.abspath(CWD)
)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=options.checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(), artifact_dir=os.path.abspath(CWD))
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(), artifact_dir=os.path.abspath(CWD))
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main():
# Create executor stub
thermos_gc_executor = ThermosGCExecutor(checkpoint_root=TaskPath.DEFAULT_CHECKPOINT_ROOT)
thermos_gc_executor.start()
# Start metrics collection
metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
metric_writer.start()
# Create driver stub
driver = MesosExecutorDriver(thermos_gc_executor)
# Start GC executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main():
# Create executor stub
thermos_gc_executor = ThermosGCExecutor(checkpoint_root=DEFAULT_CHECKPOINT_ROOT)
thermos_gc_executor.start()
# Start metrics collection
metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
metric_writer.start()
# Create driver stub
driver = MesosExecutorDriver(thermos_gc_executor)
# Start GC executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_executor = initialize(options)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
thermos_executor = initialize(options)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def main():
try:
ex_utils = EXUtils(log)
ex_utils.check_needed_envs()
ex_utils.wait_for_network_connectivity()
os.environ['LIBPROCESS_PORT'] = "0"
os.environ['LIBPROCESS_IP'] = "0.0.0.0"
driver = MesosExecutorDriver(MesosLXCExecutor(log))
log.debug('Running driver')
return 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
except Exception, e:
log.error(traceback.format_exc())
log.error(traceback.format_stack())
return 1
def main():
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# Create executor stub
thermos_gc_executor = ThermosGCExecutor(FixedPathDetector(DEFAULT_CHECKPOINT_ROOT))
thermos_gc_executor.start()
# Start metrics collection
metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH)
metric_writer.start()
# Create driver stub
driver = MesosExecutorDriver(thermos_gc_executor)
# Start GC executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def __init__(self, executor):
executor = ExecutorProxy(executor)
self.driver = MesosExecutorDriver(executor)
def shutdown(signal, frame):
self.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.stop)
class Running(object):
def __init__(self, executor):
executor = ExecutorProxy(executor)
self.driver = MesosExecutorDriver(executor)
def shutdown(signal, frame):
self.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.stop)
def run(self):
return self.driver.run()
def start(self):
status = self.driver.start()
assert status == mesos_pb2.DRIVER_RUNNING
return status
def stop(self):
return self.driver.stop()
def join(self):
return self.driver.join()
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.stop()
self.join()
if exc_type:
raise exc_type, exc_value, traceback
def main(args, options):
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# status providers:
status_providers = [HealthCheckerProvider()]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
class Running(object):
def __init__(self, executor):
executor = ExecutorProxy(executor)
self.driver = MesosExecutorDriver(executor)
def shutdown(signal, frame):
self.driver.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.driver.stop)
def run(self):
return self.driver.run()
def start(self):
status = self.driver.start()
assert status == mesos_pb2.DRIVER_RUNNING
return status
def stop(self):
logging.info("Stopping Mesos driver")
self.driver.stop()
logging.info("Joining Mesos driver")
result = self.driver.join()
logging.info("Joined Mesos driver")
if result != mesos_pb2.DRIVER_STOPPED:
raise RuntimeError("Mesos driver failed with %i", result)
def join(self):
self.driver.join()
def __enter__(self):
self.start()
return self
def __exit__(self, type, value, traceback):
self.stop()
class Running(object):
def __init__(self, executor):
executor = ExecutorProxy(executor)
self.driver = MesosExecutorDriver(executor)
def shutdown(signal, frame):
self.driver.stop()
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGTERM, shutdown)
atexit.register(self.driver.stop)
def run(self):
return self.driver.run()
def start(self):
status = self.driver.start()
assert status == mesos_pb2.DRIVER_RUNNING
return status
def stop(self):
logging.info("Stopping Mesos driver")
self.driver.stop()
logging.info("Joining Mesos driver")
result = self.driver.join()
logging.info("Joined Mesos driver")
if result != mesos_pb2.DRIVER_STOPPED:
raise RuntimeError("Mesos driver failed with %i", result)
def join(self):
self.driver.join()
def __enter__(self):
self.start()
return self
def __exit__(self, type, value, traceback):
self.stop()
def main():
driver = MesosExecutorDriver(BigDataExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print("Sent status update")
return
thread = threading.Thread(target=run_task)
thread.start()
def killTask(self, driver, taskId):
self.shutdown(driver)
def frameworkMessage(self, driver, message):
print("Ignoring framework message: %s" % message)
def shutdown(self, driver):
print("Shutting down")
sys.exit(0)
def error(self, error, message):
pass
if __name__ == "__main__":
print("Starting Launching Executor (LE)")
if len(sys.argv) == 3:
redis_host, port = sys.argv[1:3]
driver = MesosExecutorDriver(URLCrawlExecutor(redis_host, int(port)))
else:
driver = MesosExecutorDriver(URLCrawlExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def start_executor():
executor = WorkerExecutor()
driver = MesosExecutorDriver(executor)
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
print "{}".format(get_dir_addr)
task_id_msg = "task_id {}".format(task.task_id.value)
message = "{} {}".format(get_dir_addr, task_id_msg)
logging.info("Sending message: {}".format(message))
driver.sendFrameworkMessage(message)
print "Sent output file URI"
driver.sendFrameworkMessage("[EXECUTOR_STATE] {} stopped".format(\
self.executor_id))
driver.stop()
thread = threading.Thread(target=run_task)
thread.start()
thread.join()
def frameworkMessage(self, driver, message):
message_list = message.split()
if message_list[1].strip(' \t\n\r') == "abort":
logging.info("task {} aborted".format(self.task_id))
driver.sendFrameworkMessage("[EXECUTOR_STATE] {} aborted {}".format(\
self.executor_id, self.task_id))
driver.stop()
if __name__ == '__main__':
print "starting makeflow mesos executor!"
driver = MesosExecutorDriver(MakeflowMesosExecutor(sys.argv[1], sys.argv[2], sys.argv[3]))
status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
sys.exit(status)
print "Sending status update..."
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update"
return
thread = threading.Thread(target=run_task)
thread.start()
def killTask(self, driver, taskId):
shutdown(self, driver)
def frameworkMessage(self, driver, message):
print "Ignoring framework message: %s" % message
def shutdown(self, driver):
print "Shutting down"
sys.exit(0)
def error(self, error, message):
pass
if __name__ == "__main__":
print "Starting Launching Executor (LE)"
driver = MesosExecutorDriver(CrawlExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = mesos_pb2.TASK_FINISHED
driver.sendStatusUpdate(update)
print "Sent status update for task %s" % task.task_id.value
return
thread = threading.Thread(target=run_task)
thread.start()
def killTask(self, driver, taskId):
pass
def frameworkMessage(self, driver, message):
pass
def shutdown(self, driver):
pass
def error(self, error, message):
pass
if __name__ == "__main__":
print "Starting Render Executor (RE)"
local = False
if len(sys.argv) == 2 and sys.argv[1] == "--local":
local = True
driver = MesosExecutorDriver(RenderExecutor(local))
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def main():
driver = MesosExecutorDriver(ExampleExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def disconnected(self, driver):
print "HelloWorldExecutor disconnected"
def launchTask(self, driver, task):
def run_task():
def task_update(state):
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = state
driver.sendStatusUpdate(update)
print "Running Hello task %s" % task.task_id.value
task_update(mesos_pb2.TASK_RUNNING)
for i in range(0, 100):
print "%s says Hello World" % task.task_id.value
time.sleep(2)
print "Sending status update for task %s" % task.task_id.value
task_update(mesos_pb2.TASK_FINISHED)
print "Sent status update for task %s" % task.task_id.value
return
threading.Thread(target=run_task).start()
if __name__ == "__main__":
print "Starting HelloWorld Executor"
driver = MesosExecutorDriver(HelloWorldExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def start_executor():
executor = WorkerExecutor()
driver = MesosExecutorDriver(executor)
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def reregistered(self, driver, slaveInfo):
print "HelloWorldExecutor reregistered"
def disconnected(self, driver):
print "HelloWorldExecutor disconnected"
def launchTask(self, driver, task):
def run_task():
def task_update(state):
update = mesos_pb2.TaskStatus()
update.task_id.value = task.task_id.value
update.state = state
driver.sendStatusUpdate(update)
print "Running Hello task %s" % task.task_id.value
task_update(mesos_pb2.TASK_RUNNING)
for i in range(0, 100):
print "%s says Hello World" % task.task_id.value
time.sleep(2)
print "Sending status update for task %s" % task.task_id.value
task_update(mesos_pb2.TASK_FINISHED)
print "Sent status update for task %s" % task.task_id.value
return
threading.Thread(target=run_task).start()
if __name__ == "__main__":
print "Starting HelloWorld Executor"
driver = MesosExecutorDriver(HelloWorldExecutor())
sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
pass
def check_computations(self, driver):
if self.runnig_task is None:
return False
if not self.runnig_task.is_finished():
return False
logger.info("Task %s is finished. Trying to report to the master" % self.runnig_task.task['id'])
message = messages.create_message(messages.EMT_TASKFINISHED, self.runnig_task.task_repr())
driver.sendFrameworkMessage(message)
self.runnig_task = None
return True
if __name__ == "__main__":
if len(sys.argv) > 1:
time_label = sys.argv[1]
else:
time_label = utils.get_time_label()
utils.configureLogger(time_label = time_label, is_executor=True)
logger = logging.getLogger(utils.DEFAULT_LOGGER_NAME)
logger.info("Starting Launching Executor (LE)")
driver = MesosExecutorDriver(TestExecutor())
exit_code = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1
logger.info("Executor is finished")
sys.exit(exit_code)