def test_basic_as_job(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
te = AuroraExecutor(runner_provider=make_provider(tempdir),
sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver,
make_task(MESOS_JOB(task=HELLO_WORLD), instanceId=0))
te.runner_started.wait()
while te._status_manager is None:
time.sleep(0.1)
te.terminated.wait()
tm = TaskMonitor(TaskPath(root=tempdir),
task_id=HELLO_WORLD_TASK_ID)
runner_state = tm.get_state()
assert 'hello_world_hello_world-001' in runner_state.processes, (
'Could not find processes, got: %s' %
' '.join(runner_state.processes))
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 3
status_updates = [arg_tuple[0][0] for arg_tuple in updates]
assert status_updates[0].state == mesos_pb2.TASK_STARTING
assert status_updates[1].state == mesos_pb2.TASK_RUNNING
assert status_updates[2].state == mesos_pb2.TASK_FINISHED
def initialize(options):
cwd_path = os.path.abspath(CWD)
checkpoint_root = os.path.join(cwd_path,
MesosPathDetector.DEFAULT_SANDBOX_PATH)
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=checkpoint_root)
]
if options.announcer_enable:
log.warn(
'Please remove the deprecated and no-op --announcer-enable flag in scheduler config!'
)
if options.announcer_ensemble is not None:
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path,
options.announcer_allow_custom_serverset_path,
options.announcer_hostname))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
return thermos_executor
def initialize(options):
cwd_path = os.path.abspath(CWD)
checkpoint_root = os.path.join(cwd_path,
MesosPathDetector.DEFAULT_SANDBOX_PATH)
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
return thermos_executor
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=options.checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(), artifact_dir=os.path.abspath(CWD))
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(), artifact_dir=os.path.abspath(CWD))
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def test_waiting_executor():
proxy_driver = ProxyDriver()
with temporary_dir() as checkpoint_root:
te = AuroraExecutor(
runner_provider=make_provider(checkpoint_root),
sandbox_provider=DefaultTestSandboxProvider())
ExecutorTimeout(te.launched, proxy_driver, timeout=Amount(100, Time.MILLISECONDS)).start()
proxy_driver.wait_stopped()
def test_basic(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
te = AuroraExecutor(runner_provider=make_provider(tempdir), sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver, make_task(HELLO_WORLD_MTI))
te.terminated.wait()
tm = TaskMonitor(tempdir, task_id=HELLO_WORLD_TASK_ID)
runner_state = tm.get_state()
assert "hello_world_hello_world-001" in runner_state.processes, "Could not find processes, got: %s" % " ".join(
runner_state.processes
)
updates = proxy_driver.method_calls["sendStatusUpdate"]
assert len(updates) == 3
status_updates = [arg_tuple[0][0] for arg_tuple in updates]
assert status_updates[0].state == mesos_pb2.TASK_STARTING
assert status_updates[1].state == mesos_pb2.TASK_RUNNING
assert status_updates[2].state == mesos_pb2.TASK_FINISHED
def test_basic(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
te = AuroraExecutor(runner_provider=make_provider(tempdir),
sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver, make_task(HELLO_WORLD_MTI))
te.terminated.wait()
tm = TaskMonitor(tempdir, task_id=HELLO_WORLD_TASK_ID)
runner_state = tm.get_state()
assert 'hello_world_hello_world-001' in runner_state.processes, (
'Could not find processes, got: %s' %
' '.join(runner_state.processes))
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 3
status_updates = [arg_tuple[0][0] for arg_tuple in updates]
assert status_updates[0].state == mesos_pb2.TASK_STARTING
assert status_updates[1].state == mesos_pb2.TASK_RUNNING
assert status_updates[2].state == mesos_pb2.TASK_FINISHED
def test_basic_as_job(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
te = AuroraExecutor(
runner_provider=make_provider(tempdir),
sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver, make_task(MESOS_JOB(task=HELLO_WORLD), instanceId=0))
te.runner_started.wait()
while te._status_manager is None:
time.sleep(0.1)
te.terminated.wait()
tm = TaskMonitor(tempdir, task_id=HELLO_WORLD_TASK_ID)
runner_state = tm.get_state()
assert 'hello_world_hello_world-001' in runner_state.processes, (
'Could not find processes, got: %s' % ' '.join(runner_state.processes))
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 3
status_updates = [arg_tuple[0][0] for arg_tuple in updates]
assert status_updates[0].state == mesos_pb2.TASK_STARTING
assert status_updates[1].state == mesos_pb2.TASK_RUNNING
assert status_updates[2].state == mesos_pb2.TASK_FINISHED
def main(args, options):
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# status providers:
status_providers = [HealthCheckerProvider()]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def initialize(options):
cwd_path = os.path.abspath(CWD)
checkpoint_root = os.path.join(cwd_path,
MesosPathDetector.DEFAULT_SANDBOX_PATH)
# status providers:
status_providers = [
HealthCheckerProvider(
nosetuid_health_checks=options.nosetuid_health_checks,
mesos_containerizer_path=options.mesos_containerizer_path),
ResourceManagerProvider(checkpoint_root=checkpoint_root)
]
if options.announcer_ensemble is not None:
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path,
options.announcer_allow_custom_serverset_path,
options.announcer_hostname,
make_zk_auth(options.announcer_zookeeper_auth_config)))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env,
mesos_containerizer_path=options.mesos_containerizer_path)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user),
no_sandbox_create_user=options.no_create_user,
sandbox_mount_point=options.sandbox_mount_point)
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env,
mesos_containerizer_path=options.mesos_containerizer_path)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
no_sandbox_create_user=options.no_create_user,
sandbox_mount_point=options.sandbox_mount_point)
return thermos_executor
