def initialize(options): cwd_path = os.path.abspath(CWD) checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=checkpoint_root) ] if options.announcer_enable: log.warn('Please remove the deprecated and no-op --announcer-enable flag in scheduler config!') if options.announcer_ensemble is not None: status_providers.append(DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path, options.announcer_allow_custom_serverset_path, options.announcer_hostname, make_zk_auth(options.announcer_zookeeper_auth_config) )) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env ) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user) ) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env ) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers ) return thermos_executor
def initialize(options): cwd_path = os.path.abspath(CWD) checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=checkpoint_root) ] if options.announcer_enable: log.warn( 'Please remove the deprecated and no-op --announcer-enable flag in scheduler config!' ) if options.announcer_ensemble is not None: status_providers.append( DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path, options.announcer_allow_custom_serverset_path, options.announcer_hostname)) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider( options.execute_as_user)) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers) return thermos_executor
def initialize(options): cwd_path = os.path.abspath(CWD) checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=checkpoint_root) ] if options.announcer_enable: if options.announcer_ensemble is None: app.error('Must specify --announcer-ensemble if the announcer is enabled.') status_providers.append(DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path, options.announcer_allow_custom_serverset_path )) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env ) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user) ) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env ) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers ) return thermos_executor
def initialize(options): cwd_path = os.path.abspath(CWD) checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=checkpoint_root) ] if options.announcer_enable: if options.announcer_ensemble is None: app.error( 'Must specify --announcer-ensemble if the announcer is enabled.' ) status_providers.append( DefaultAnnouncerCheckerProvider(options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider( options.execute_as_user)) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers) return thermos_executor
def main(args, options): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=options.checkpoint_root) ] if options.announcer_enable: if options.announcer_ensemble is None: app.error('Must specify --announcer-ensemble if the announcer is enabled.') status_providers.append(DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD) ) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user) ) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD) ) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers ) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def main(args, options): if MesosExecutorDriver is None: app.error('Could not load MesosExecutorDriver!') # status providers: status_providers = [ HealthCheckerProvider(), ResourceManagerProvider(checkpoint_root=options.checkpoint_root) ] if options.announcer_enable: if options.announcer_ensemble is None: app.error( 'Must specify --announcer-ensemble if the announcer is enabled.' ) status_providers.append( DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path)) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD)) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider( options.execute_as_user)) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), artifact_dir=os.path.abspath(CWD)) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers) # Create driver stub driver = MesosExecutorDriver(thermos_executor) # This is an ephemeral executor -- shutdown if we receive no tasks within a certain # time period ExecutorTimeout(thermos_executor.launched, driver).start() # Start executor driver.run() log.info('MesosExecutorDriver.run() has finished.')
def initialize(options): cwd_path = os.path.abspath(CWD) checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) # status providers: status_providers = [ HealthCheckerProvider( nosetuid_health_checks=options.nosetuid_health_checks, mesos_containerizer_path=options.mesos_containerizer_path), ResourceManagerProvider(checkpoint_root=checkpoint_root) ] if options.announcer_ensemble is not None: status_providers.append( DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path, options.announcer_allow_custom_serverset_path, options.announcer_hostname, make_zk_auth(options.announcer_zookeeper_auth_config))) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env, mesos_containerizer_path=options.mesos_containerizer_path) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider( options.execute_as_user), no_sandbox_create_user=options.no_create_user, sandbox_mount_point=options.sandbox_mount_point) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env, mesos_containerizer_path=options.mesos_containerizer_path) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, no_sandbox_create_user=options.no_create_user, sandbox_mount_point=options.sandbox_mount_point) return thermos_executor
def initialize(options): cwd_path = os.path.abspath(CWD) checkpoint_root = os.path.join(cwd_path, MesosPathDetector.DEFAULT_SANDBOX_PATH) # status providers: status_providers = [ HealthCheckerProvider( nosetuid_health_checks=options.nosetuid_health_checks, mesos_containerizer_path=options.mesos_containerizer_path), ResourceManagerProvider(checkpoint_root=checkpoint_root) ] if options.announcer_ensemble is not None: status_providers.append(DefaultAnnouncerCheckerProvider( options.announcer_ensemble, options.announcer_serverset_path, options.announcer_allow_custom_serverset_path, options.announcer_hostname, make_zk_auth(options.announcer_zookeeper_auth_config) )) # Create executor stub if options.execute_as_user or options.nosetuid: # If nosetuid is set, execute_as_user is also None thermos_runner_provider = UserOverrideThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env, mesos_containerizer_path=options.mesos_containerizer_path ) thermos_runner_provider.set_role(None) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, sandbox_provider=UserOverrideDirectorySandboxProvider(options.execute_as_user), no_sandbox_create_user=options.no_create_user, sandbox_mount_point=options.sandbox_mount_point, stop_timeout_in_secs=options.stop_timeout_in_secs ) else: thermos_runner_provider = DefaultThermosTaskRunnerProvider( dump_runner_pex(), checkpoint_root, artifact_dir=cwd_path, process_logger_destination=options.runner_logger_destination, process_logger_mode=options.runner_logger_mode, rotate_log_size_mb=options.runner_rotate_log_size_mb, rotate_log_backups=options.runner_rotate_log_backups, preserve_env=options.preserve_env, mesos_containerizer_path=options.mesos_containerizer_path ) thermos_executor = AuroraExecutor( runner_provider=thermos_runner_provider, status_providers=status_providers, no_sandbox_create_user=options.no_create_user, sandbox_mount_point=options.sandbox_mount_point, stop_timeout_in_secs=options.stop_timeout_in_secs ) return thermos_executor