def test_announcer_provider_with_timeout(mock_client_provider,
mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
client_connect_event = threading.Event()
mock_client.start_async.return_value = client_connect_event
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com',
root='/aurora')
job = make_job('aurora',
'prod',
'proxy',
'primary',
portmap={
'http': 80,
'admin': 'primary'
})
health_check_config = HealthCheckConfig(initial_interval_secs=0.1,
interval_secs=0.1)
job = job(health_check_config=health_check_config)
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
checker = dap.from_assigned_task(assigned_task, None)
mock_client.start_async.assert_called_once_with()
mock_serverset_provider.assert_called_once_with(
mock_client, '/aurora/aurora/prod/proxy')
checker.start()
checker.start_event.wait()
assert checker.status is not None
def test_default_announcer_provider(mock_client_provider,
mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com',
root='/aurora')
job = make_job('aurora',
'prod',
'proxy',
'primary',
portmap={
'http': 80,
'admin': 'primary'
})
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
checker = dap.from_assigned_task(assigned_task, None)
mock_client.start_async.assert_called_once_with()
mock_serverset_provider.assert_called_once_with(
mock_client, '/aurora/aurora/prod/proxy')
assert checker.name() == 'announcer'
assert checker.status is None
def test_announcer_provider_with_acl(mock_client_provider,
mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
zk_auth = ZkAuth(auth=[Auth(scheme='s', credential='ca')],
acl=[
Access(scheme='s',
credential='cl',
permissions=Permissions(create=True))
])
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', '', False,
None, zk_auth)
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={})
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
dap.from_assigned_task(assigned_task, None)
mock_client_provider.assert_called_once_with(
'zookeeper.example.com',
connection_retry=dap.DEFAULT_RETRY_POLICY,
auth_data=[('s', 'ca')],
default_acl=[ACL(KazooPermissions.CREATE, Id('s', 'cl'))])
def test_announcer_provider_with_hostname(endpoint_mock_provider,
mock_client_provider, mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', '/aurora', False, '10.2.3.4')
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={})
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
dap.from_assigned_task(assigned_task, None)
assert endpoint_mock_provider.mock_calls == [call('10.2.3.4', 12345), call('10.2.3.4', 12345)]
def test_announcer_provider_with_hostname(endpoint_mock_provider,
mock_client_provider, mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', '/aurora', False, '10.2.3.4')
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={})
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
dap.from_assigned_task(assigned_task, None)
assert endpoint_mock_provider.mock_calls == [call('10.2.3.4', 12345), call('10.2.3.4', 12345)]
def test_default_announcer_provider(mock_client_provider, mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', root='/aurora')
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={'http': 80, 'admin': 'primary'})
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
checker = dap.from_assigned_task(assigned_task, None)
mock_client.start_async.assert_called_once_with()
mock_serverset_provider.assert_called_once_with(mock_client, '/aurora/aurora/prod/proxy')
assert checker.name() == 'announcer'
assert checker.status is None
def test_default_announcer_provider(mock_client_provider, mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider("zookeeper.example.com", root="/aurora")
job = make_job("aurora", "prod", "proxy", "primary", portmap={"http": 80, "admin": "primary"})
assigned_task = make_assigned_task(job, assigned_ports={"primary": 12345})
checker = dap.from_assigned_task(assigned_task, None)
mock_client.start_async.assert_called_once_with()
mock_serverset_provider.assert_called_once_with(mock_client, "/aurora/aurora/prod/proxy")
assert checker.name() == "announcer"
assert checker.status is None
def test_default_announcer_provider_without_announce():
from pystachio import Empty
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={})
job = job(announce=Empty)
assigned_task = make_assigned_task(job)
assert DefaultAnnouncerCheckerProvider('foo.bar').from_assigned_task(assigned_task, None) is None
def initialize(options):
cwd_path = os.path.abspath(CWD)
checkpoint_root = os.path.join(cwd_path,
MesosPathDetector.DEFAULT_SANDBOX_PATH)
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=checkpoint_root)
]
if options.announcer_enable:
log.warn(
'Please remove the deprecated and no-op --announcer-enable flag in scheduler config!'
)
if options.announcer_ensemble is not None:
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path,
options.announcer_allow_custom_serverset_path,
options.announcer_hostname))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
return thermos_executor
def test_announcer_provider_with_acl(mock_client_provider, mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
zk_auth = ZkAuth(auth=[Auth(scheme='s', credential='ca')],
acl=[Access(scheme='s', credential='cl', permissions=Permissions(create=True))])
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', '', False, None, zk_auth)
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={})
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
dap.from_assigned_task(assigned_task, None)
mock_client_provider.assert_called_once_with('zookeeper.example.com',
connection_retry=dap.DEFAULT_RETRY_POLICY,
auth_data=[('s', 'ca')],
default_acl=[ACL(KazooPermissions.CREATE,
Id('s', 'cl'))])
def initialize(options):
cwd_path = os.path.abspath(CWD)
checkpoint_root = os.path.join(cwd_path,
MesosPathDetector.DEFAULT_SANDBOX_PATH)
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
return thermos_executor
def main(args, options):
if MesosExecutorDriver is None:
app.error('Could not load MesosExecutorDriver!')
# status providers:
status_providers = [
HealthCheckerProvider(),
ResourceManagerProvider(checkpoint_root=options.checkpoint_root)
]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(), artifact_dir=os.path.abspath(CWD))
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user))
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(), artifact_dir=os.path.abspath(CWD))
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def test_announcer_provider_with_timeout(mock_client_provider, mock_serverset_provider):
mock_client = create_autospec(spec=KazooClient, instance=True)
mock_client_provider.return_value = mock_client
client_connect_event = threading.Event()
mock_client.start_async.return_value = client_connect_event
mock_serverset = create_autospec(spec=ServerSet, instance=True)
mock_serverset_provider.return_value = mock_serverset
dap = DefaultAnnouncerCheckerProvider('zookeeper.example.com', root='/aurora')
job = make_job('aurora', 'prod', 'proxy', 'primary', portmap={'http': 80, 'admin': 'primary'})
health_check_config = HealthCheckConfig(initial_interval_secs=0.1, interval_secs=0.1)
job = job(health_check_config=health_check_config)
assigned_task = make_assigned_task(job, assigned_ports={'primary': 12345})
checker = dap.from_assigned_task(assigned_task, None)
mock_client.start_async.assert_called_once_with()
mock_serverset_provider.assert_called_once_with(mock_client, '/aurora/aurora/prod/proxy')
checker.start()
checker.start_event.wait()
assert checker.status is not None
def main(args, options):
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
artifact_dir=os.path.realpath('.'),
)
# status providers:
status_providers = [HealthCheckerProvider()]
if options.announcer_enable:
if options.announcer_ensemble is None:
app.error(
'Must specify --announcer-ensemble if the announcer is enabled.'
)
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble,
options.announcer_serverset_path))
# Create executor stub
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
)
# Create driver stub
driver = MesosExecutorDriver(thermos_executor)
# This is an ephemeral executor -- shutdown if we receive no tasks within a certain
# time period
ExecutorTimeout(thermos_executor.launched, driver).start()
# Start executor
driver.run()
log.info('MesosExecutorDriver.run() has finished.')
def initialize(options):
cwd_path = os.path.abspath(CWD)
checkpoint_root = os.path.join(cwd_path,
MesosPathDetector.DEFAULT_SANDBOX_PATH)
# status providers:
status_providers = [
HealthCheckerProvider(
nosetuid_health_checks=options.nosetuid_health_checks,
mesos_containerizer_path=options.mesos_containerizer_path),
ResourceManagerProvider(checkpoint_root=checkpoint_root)
]
if options.announcer_ensemble is not None:
status_providers.append(
DefaultAnnouncerCheckerProvider(
options.announcer_ensemble, options.announcer_serverset_path,
options.announcer_allow_custom_serverset_path,
options.announcer_hostname,
make_zk_auth(options.announcer_zookeeper_auth_config)))
# Create executor stub
if options.execute_as_user or options.nosetuid:
# If nosetuid is set, execute_as_user is also None
thermos_runner_provider = UserOverrideThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env,
mesos_containerizer_path=options.mesos_containerizer_path)
thermos_runner_provider.set_role(None)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
sandbox_provider=UserOverrideDirectorySandboxProvider(
options.execute_as_user),
no_sandbox_create_user=options.no_create_user,
sandbox_mount_point=options.sandbox_mount_point)
else:
thermos_runner_provider = DefaultThermosTaskRunnerProvider(
dump_runner_pex(),
checkpoint_root,
artifact_dir=cwd_path,
process_logger_destination=options.runner_logger_destination,
process_logger_mode=options.runner_logger_mode,
rotate_log_size_mb=options.runner_rotate_log_size_mb,
rotate_log_backups=options.runner_rotate_log_backups,
preserve_env=options.preserve_env,
mesos_containerizer_path=options.mesos_containerizer_path)
thermos_executor = AuroraExecutor(
runner_provider=thermos_runner_provider,
status_providers=status_providers,
no_sandbox_create_user=options.no_create_user,
sandbox_mount_point=options.sandbox_mount_point)
return thermos_executor
