class Running(object): def __init__(self, executor): executor = ExecutorProxy(executor) self.driver = MesosExecutorDriver(executor) def shutdown(signal, frame): self.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.stop) def run(self): return self.driver.run() def start(self): status = self.driver.start() assert status == mesos_pb2.DRIVER_RUNNING return status def stop(self): return self.driver.stop() def join(self): return self.driver.join() def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_value, traceback): self.stop() self.join() if exc_type: raise exc_type, exc_value, traceback
def main(args, options): thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.realpath('.'), ) # status providers: status_providers = [HealthCheckerProvider()] if options.announcer_enable: if options.announcer_ensemble is None: app.error('Must specify --announcer-ensemble if the announcer is enabled.') status_providers.append(DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, ) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def __init__(self, executor): executor = ExecutorProxy(executor) self.driver = MesosExecutorDriver(executor) def shutdown(signal, frame): self.driver.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.driver.stop)
def main(args, options): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=options.checkpoint_root) ] if options.announcer_enable: if options.announcer_ensemble is None: app.error('Must specify --announcer-ensemble if the announcer is enabled.') status_providers.append(DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD) ) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user) ) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD) ) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers ) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def main(args, options): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=options.checkpoint_root) ] if options.announcer_enable: if options.announcer_ensemble is None: app.error( 'Must specify --announcer-ensemble if the announcer is enabled.' ) status_providers.append( DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD)) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider( options.execute_as_user)) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD)) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def main(): # Create executor stub thermos_gc_executor = ThermosGCExecutor(checkpoint_root=TaskPath.DEFAULT_CHECKPOINT_ROOT) thermos_gc_executor.start() # Start metrics collection metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH) metric_writer.start() # Create driver stub driver = MesosExecutorDriver(thermos_gc_executor) # Start GC executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def main(): # Create executor stub thermos_gc_executor = ThermosGCExecutor(checkpoint_root=DEFAULT_CHECKPOINT_ROOT) thermos_gc_executor.start() # Start metrics collection metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH) metric_writer.start() # Create driver stub driver = MesosExecutorDriver(thermos_gc_executor) # Start GC executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def main(args, options): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') thermos_executor = initialize(options) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def main(): try: ex_utils = EXUtils(log) ex_utils.check_needed_envs() ex_utils.wait_for_network_connectivity() os.environ['LIBPROCESS_PORT'] = "0" os.environ['LIBPROCESS_IP'] = "0.0.0.0" driver = MesosExecutorDriver(MesosLXCExecutor(log)) log.debug('Running driver') return 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 except Exception, e: log.error(traceback.format_exc()) log.error(traceback.format_stack()) return 1
def main(): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') # Create executor stub thermos_gc_executor = ThermosGCExecutor(FixedPathDetector(DEFAULT_CHECKPOINT_ROOT)) thermos_gc_executor.start() # Start metrics collection metric_writer = DiskMetricWriter(thermos_gc_executor.metrics, ExecutorDetector.VARS_PATH) metric_writer.start() # Create driver stub driver = MesosExecutorDriver(thermos_gc_executor) # Start GC executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def __init__(self, executor): executor = ExecutorProxy(executor) self.driver = MesosExecutorDriver(executor) def shutdown(signal, frame): self.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.stop)
def main(args, options): thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.realpath('.'), ) # status providers: status_providers = [HealthCheckerProvider()] if options.announcer_enable: if options.announcer_ensemble is None: app.error( 'Must specify --announcer-ensemble if the announcer is enabled.' ) status_providers.append( DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, ) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
class Running(object): def __init__(self, executor): executor = ExecutorProxy(executor) self.driver = MesosExecutorDriver(executor) def shutdown(signal, frame): self.driver.stop() signal.signal(signal.SIGINT, shutdown) signal.signal(signal.SIGTERM, shutdown) atexit.register(self.driver.stop) def run(self): return self.driver.run() def start(self): status = self.driver.start() assert status == mesos_pb2.DRIVER_RUNNING return status def stop(self): logging.info("Stopping Mesos driver") self.driver.stop() logging.info("Joining Mesos driver") result = self.driver.join() logging.info("Joined Mesos driver") if result != mesos_pb2.DRIVER_STOPPED: raise RuntimeError("Mesos driver failed with %i", result) def join(self): self.driver.join() def __enter__(self): self.start() return self def __exit__(self, type, value, traceback): self.stop()
def main(): driver = MesosExecutorDriver(BigDataExecutor()) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print("Sent status update") return thread = threading.Thread(target=run_task) thread.start() def killTask(self, driver, taskId): self.shutdown(driver) def frameworkMessage(self, driver, message): print("Ignoring framework message: %s" % message) def shutdown(self, driver): print("Shutting down") sys.exit(0) def error(self, error, message): pass if __name__ == "__main__": print("Starting Launching Executor (LE)") if len(sys.argv) == 3: redis_host, port = sys.argv[1:3] driver = MesosExecutorDriver(URLCrawlExecutor(redis_host, int(port))) else: driver = MesosExecutorDriver(URLCrawlExecutor()) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def start_executor(): executor = WorkerExecutor() driver = MesosExecutorDriver(executor) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
print "{}".format(get_dir_addr) task_id_msg = "task_id {}".format(task.task_id.value) message = "{} {}".format(get_dir_addr, task_id_msg) logging.info("Sending message: {}".format(message)) driver.sendFrameworkMessage(message) print "Sent output file URI" driver.sendFrameworkMessage("[EXECUTOR_STATE] {} stopped".format(\ self.executor_id)) driver.stop() thread = threading.Thread(target=run_task) thread.start() thread.join() def frameworkMessage(self, driver, message): message_list = message.split() if message_list[1].strip(' \t\n\r') == "abort": logging.info("task {} aborted".format(self.task_id)) driver.sendFrameworkMessage("[EXECUTOR_STATE] {} aborted {}".format(\ self.executor_id, self.task_id)) driver.stop() if __name__ == '__main__': print "starting makeflow mesos executor!" driver = MesosExecutorDriver(MakeflowMesosExecutor(sys.argv[1], sys.argv[2], sys.argv[3])) status = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 sys.exit(status)
print "Sending status update..." update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update" return thread = threading.Thread(target=run_task) thread.start() def killTask(self, driver, taskId): shutdown(self, driver) def frameworkMessage(self, driver, message): print "Ignoring framework message: %s" % message def shutdown(self, driver): print "Shutting down" sys.exit(0) def error(self, error, message): pass if __name__ == "__main__": print "Starting Launching Executor (LE)" driver = MesosExecutorDriver(CrawlExecutor()) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = mesos_pb2.TASK_FINISHED driver.sendStatusUpdate(update) print "Sent status update for task %s" % task.task_id.value return thread = threading.Thread(target=run_task) thread.start() def killTask(self, driver, taskId): pass def frameworkMessage(self, driver, message): pass def shutdown(self, driver): pass def error(self, error, message): pass if __name__ == "__main__": print "Starting Render Executor (RE)" local = False if len(sys.argv) == 2 and sys.argv[1] == "--local": local = True driver = MesosExecutorDriver(RenderExecutor(local)) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def main(): driver = MesosExecutorDriver(ExampleExecutor()) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def disconnected(self, driver): print "HelloWorldExecutor disconnected" def launchTask(self, driver, task): def run_task(): def task_update(state): update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = state driver.sendStatusUpdate(update) print "Running Hello task %s" % task.task_id.value task_update(mesos_pb2.TASK_RUNNING) for i in range(0, 100): print "%s says Hello World" % task.task_id.value time.sleep(2) print "Sending status update for task %s" % task.task_id.value task_update(mesos_pb2.TASK_FINISHED) print "Sent status update for task %s" % task.task_id.value return threading.Thread(target=run_task).start() if __name__ == "__main__": print "Starting HelloWorld Executor" driver = MesosExecutorDriver(HelloWorldExecutor()) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
def reregistered(self, driver, slaveInfo): print "HelloWorldExecutor reregistered" def disconnected(self, driver): print "HelloWorldExecutor disconnected" def launchTask(self, driver, task): def run_task(): def task_update(state): update = mesos_pb2.TaskStatus() update.task_id.value = task.task_id.value update.state = state driver.sendStatusUpdate(update) print "Running Hello task %s" % task.task_id.value task_update(mesos_pb2.TASK_RUNNING) for i in range(0, 100): print "%s says Hello World" % task.task_id.value time.sleep(2) print "Sending status update for task %s" % task.task_id.value task_update(mesos_pb2.TASK_FINISHED) print "Sent status update for task %s" % task.task_id.value return threading.Thread(target=run_task).start() if __name__ == "__main__": print "Starting HelloWorld Executor" driver = MesosExecutorDriver(HelloWorldExecutor()) sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)
pass def check_computations(self, driver): if self.runnig_task is None: return False if not self.runnig_task.is_finished(): return False logger.info("Task %s is finished. Trying to report to the master" % self.runnig_task.task['id']) message = messages.create_message(messages.EMT_TASKFINISHED, self.runnig_task.task_repr()) driver.sendFrameworkMessage(message) self.runnig_task = None return True if __name__ == "__main__": if len(sys.argv) > 1: time_label = sys.argv[1] else: time_label = utils.get_time_label() utils.configureLogger(time_label = time_label, is_executor=True) logger = logging.getLogger(utils.DEFAULT_LOGGER_NAME) logger.info("Starting Launching Executor (LE)") driver = MesosExecutorDriver(TestExecutor()) exit_code = 0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1 logger.info("Executor is finished") sys.exit(exit_code)