def test_health_check_ok(self, mock_sub):
timeout = 30
cmd = 'success cmd'
shell = ShellHealthCheck(cmd, timeout_secs=timeout)
success, msg = shell()
self.assertTrue(success)
self.assertIsNone(msg)
mock_sub.assert_called_once_with(['success', 'cmd'], timeout=30)
def from_assigned_task(self, assigned_task, sandbox):
"""
:param assigned_task:
:param sandbox:
:return: Instance of a HealthChecker.
"""
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
health_check_config = mesos_task.health_check_config().get()
health_checker = health_check_config.get('health_checker', {})
timeout_secs = health_check_config.get('timeout_secs')
if SHELL_HEALTH_CHECK in health_checker:
shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command')
# Filling in variables eg thermos.ports[http] that could have been passed in as part of
# shell_command.
interpolated_command = HealthCheckerProvider.interpolate_cmd(
task=assigned_task,
cmd=shell_command
)
# If we do not want user which is job's role to execute the health shell check
# --nosetuid-health-checks should be passed in as an argument to the executor.
demote_to_job_role_user = None
if not self.nosetuid_health_checks:
pw_entry = pwd.getpwnam(assigned_task.task.job.role)
def demote_to_job_role_user():
os.setgid(pw_entry.pw_gid)
os.setuid(pw_entry.pw_uid)
shell_signaler = ShellHealthCheck(cmd=interpolated_command,
preexec_fn=demote_to_job_role_user,
timeout_secs=timeout_secs)
a_health_checker = lambda: shell_signaler()
else:
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
if 'health' not in portmap:
return None
http_config = health_checker.get(HTTP_HEALTH_CHECK, {})
http_endpoint = http_config.get('endpoint')
http_expected_response = http_config.get('expected_response')
http_expected_response_code = http_config.get('expected_response_code')
http_signaler = HttpSignaler(
portmap['health'],
timeout_secs=timeout_secs)
a_health_checker = lambda: http_signaler(
endpoint=http_endpoint,
expected_response=http_expected_response,
expected_response_code=http_expected_response_code
)
health_checker = HealthChecker(
a_health_checker,
sandbox,
interval_secs=health_check_config.get('interval_secs'),
initial_interval_secs=health_check_config.get('initial_interval_secs'),
max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
return health_checker
def test_health_check_ok(self, mock_popen):
shell = ShellHealthCheck('cmd', timeout_secs=30)
success, msg = shell()
self.assertTrue(success)
self.assertIsNone(msg)
mock_popen.assert_called_once_with('cmd',
shell=True,
timeout=30,
preexec_fn=mock.ANY)
def test_health_check_value_error(self, mock_popen):
# Invalid commmand passed in raises ValueError.
mock_popen.side_effect = ValueError('Could not read command.')
timeout = 10
shell = ShellHealthCheck('cmd', timeout_secs=timeout)
success, msg = shell()
mock_popen.assert_called_once_with('cmd', shell=True, timeout=10, preexec_fn=mock.ANY)
self.assertFalse(success)
self.assertEqual(msg, 'Invalid commmand.')
def test_health_check_os_error(self, mock_popen):
# Fail due to command not existing.
mock_popen.side_effect = OSError(1, 'failed')
shell = ShellHealthCheck('cmd', timeout_secs=30)
success, msg = shell()
mock_popen.assert_called_once_with('cmd', shell=True, timeout=30, preexec_fn=mock.ANY)
self.assertFalse(success)
self.assertEqual(msg, 'OSError: failed')
def test_health_check_timeout(self, mock_popen):
# Fail due to command returning a non-0 exit status.
mock_popen.side_effect = subprocess.TimeoutExpired('failed', timeout=30)
shell = ShellHealthCheck('cmd', timeout_secs=30)
success, msg = shell()
mock_popen.assert_called_once_with('cmd', shell=True, timeout=30, preexec_fn=mock.ANY)
self.assertFalse(success)
self.assertEqual(msg, 'Health check timed out.')
def test_health_check_ok(self, mock_popen):
shell = ShellHealthCheck(raw_cmd='cmd',
wrapped_cmd='wrapped-cmd',
timeout_secs=30)
success, msg = shell()
self.assertTrue(success)
self.assertIsNone(msg)
mock_popen.assert_called_once_with('wrapped-cmd',
timeout=30,
preexec_fn=mock.ANY,
stderr=STDOUT)
def test_health_check_failed(self, mock_popen):
cmd = 'failed'
# Fail due to command returning a non-0 exit status.
mock_popen.side_effect = subprocess.CalledProcessError(1, cmd)
shell = ShellHealthCheck(cmd, timeout_secs=30)
success, msg = shell()
mock_popen.assert_called_once_with(cmd, shell=True, timeout=30, preexec_fn=mock.ANY)
self.assertFalse(success)
self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def test_health_check_value_error(self, mock_sub):
timeout = 30
# Invalid commmand passed in raises ValueError.
mock_sub.side_effect = ValueError('Could not read command.')
cmd = 'defensive cmd'
timeout = 10
shell = ShellHealthCheck(cmd, timeout_secs=timeout)
success, msg = shell()
mock_sub.assert_called_once_with(['defensive', 'cmd'], timeout=10)
self.assertFalse(success)
self.assertEqual(msg, 'Invalid commmand.')
def test_health_check_os_error(self, mock_sub):
timeout = 30
# Fail due to command not existing.
mock_sub.side_effect = OSError(1, 'failed')
cmd = 'cmd to not exist'
shell = ShellHealthCheck(cmd, timeout_secs=timeout)
success, msg = shell()
mock_sub.assert_called_once_with(['cmd', 'to', 'not', 'exist'],
timeout=30)
self.assertFalse(success)
self.assertEqual(msg, 'OSError: failed')
def test_health_check_failed(self, mock_sub):
timeout = 30
# Fail due to command returning a non-0 exit status.
mock_sub.side_effect = subprocess.CalledProcessError(1, 'failed')
cmd = 'cmd to fail'
shell = ShellHealthCheck(cmd, timeout_secs=timeout)
success, msg = shell()
mock_sub.assert_called_once_with(['cmd', 'to', 'fail'], timeout=30)
self.assertFalse(success)
self.assertEqual(msg,
"Command 'failed' returned non-zero exit status 1")
def test_health_check_failed_with_wrapper(self, mock_popen):
cmd = 'failed'
mock_popen.side_effect = subprocess.CalledProcessError(1, cmd)
shell = ShellHealthCheck(cmd, timeout_secs=30, wrapper_fn=lambda c: 'wrapped: %s' % c)
success, msg = shell()
self.assertEqual(
mock_popen.mock_calls,
[mock.call('wrapped: %s' % cmd, shell=True, timeout=30, preexec_fn=mock.ANY)])
self.assertFalse(success)
self.assertEqual(msg, "Command 'failed' returned non-zero exit status 1")
def from_assigned_task(self, assigned_task, sandbox):
"""
:param assigned_task:
:param sandbox:
:return: Instance of a HealthChecker.
"""
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
health_check_config = mesos_task.health_check_config().get()
health_checker = health_check_config.get('health_checker', {})
timeout_secs = health_check_config.get('timeout_secs')
if SHELL_HEALTH_CHECK in health_checker:
shell_command = health_checker.get(SHELL_HEALTH_CHECK,
{}).get('shell_command')
shell_signaler = ShellHealthCheck(cmd=shell_command,
timeout_secs=timeout_secs)
a_health_checker = lambda: shell_signaler()
else:
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
if 'health' not in portmap:
return None
if HTTP_HEALTH_CHECK in health_checker:
# Assume user has already switched over to the new config since we found the key.
http_config = health_checker.get(HTTP_HEALTH_CHECK, {})
http_endpoint = http_config.get('endpoint')
http_expected_response = http_config.get('expected_response')
http_expected_response_code = http_config.get(
'expected_response_code')
else:
# TODO (AURORA-1563): Remove this clause after we deprecate support for following keys
# directly in HealthCheckConfig
http_endpoint = health_check_config.get('endpoint')
http_expected_response = health_check_config.get(
'expected_response')
http_expected_response_code = health_check_config.get(
'expected_response_code')
http_signaler = HttpSignaler(portmap['health'],
timeout_secs=timeout_secs)
a_health_checker = lambda: http_signaler(
endpoint=http_endpoint,
expected_response=http_expected_response,
expected_response_code=http_expected_response_code)
health_checker = HealthChecker(
a_health_checker,
sandbox,
interval_secs=health_check_config.get('interval_secs'),
initial_interval_secs=health_check_config.get(
'initial_interval_secs'),
max_consecutive_failures=health_check_config.get(
'max_consecutive_failures'))
return health_checker
def from_assigned_task(self, assigned_task, sandbox):
"""
:param assigned_task:
:param sandbox:
:return: Instance of a HealthChecker.
"""
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
health_check_config = mesos_task.health_check_config().get()
health_checker = health_check_config.get('health_checker', {})
timeout_secs = health_check_config.get('timeout_secs')
if SHELL_HEALTH_CHECK in health_checker:
shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command')
# Filling in variables eg thermos.ports[http] that could have been passed in as part of
# shell_command.
interpolated_command = HealthCheckerProvider.interpolate_cmd(
task=assigned_task,
cmd=shell_command
)
shell_signaler = ShellHealthCheck(
cmd=interpolated_command,
timeout_secs=timeout_secs,
)
a_health_checker = lambda: shell_signaler()
else:
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
if 'health' not in portmap:
return None
http_config = health_checker.get(HTTP_HEALTH_CHECK, {})
http_endpoint = http_config.get('endpoint')
http_expected_response = http_config.get('expected_response')
http_expected_response_code = http_config.get('expected_response_code')
http_signaler = HttpSignaler(
portmap['health'],
timeout_secs=timeout_secs)
a_health_checker = lambda: http_signaler(
endpoint=http_endpoint,
expected_response=http_expected_response,
expected_response_code=http_expected_response_code
)
health_checker = HealthChecker(
a_health_checker,
sandbox,
interval_secs=health_check_config.get('interval_secs'),
initial_interval_secs=health_check_config.get('initial_interval_secs'),
max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
return health_checker
def test_health_check_failed(self, mock_popen):
cmd = 'failed'
wrapped_cmd = 'wrapped-failed'
# Fail due to command returning a non-0 exit status.
mock_popen.side_effect = subprocess.CalledProcessError(
1, wrapped_cmd, output='No file.')
shell = ShellHealthCheck(raw_cmd=cmd,
wrapped_cmd=wrapped_cmd,
timeout_secs=30)
success, msg = shell()
mock_popen.assert_called_once_with(wrapped_cmd,
timeout=30,
preexec_fn=mock.ANY,
stderr=STDOUT)
self.assertFalse(success)
self.assertEqual(
msg,
"Command 'failed' returned non-zero exit status 1 with output 'No file.'"
)
def from_assigned_task(self, assigned_task, sandbox):
"""
:param assigned_task:
:param sandbox:
:return: Instance of a HealthChecker.
"""
mesos_task = mesos_task_instance_from_assigned_task(assigned_task)
health_check_config = mesos_task.health_check_config().get()
health_checker = health_check_config.get('health_checker', {})
timeout_secs = health_check_config.get('timeout_secs')
if SHELL_HEALTH_CHECK in health_checker:
shell_command = health_checker.get(SHELL_HEALTH_CHECK, {}).get('shell_command')
# Filling in variables e.g. thermos.ports[http] that could have been passed in as part of
# shell_command.
interpolated_command = HealthCheckerProvider.interpolate_cmd(
task=assigned_task,
cmd=shell_command)
# If we do not want the health check to execute as the user from the job's role
# --nosetuid-health-checks should be passed as an argument to the executor.
demote_to_job_role_user = None
if not self._nosetuid_health_checks and not sandbox.is_filesystem_image:
pw_entry = pwd.getpwnam(assigned_task.task.job.role)
def demote_to_job_role_user():
os.setgid(pw_entry.pw_gid)
os.setuid(pw_entry.pw_uid)
# If the task is executing in an isolated filesystem we'll want to wrap the health check
# command within a mesos-containerizer invocation so that it's executed within that
# filesystem.
wrapper = None
if sandbox.is_filesystem_image:
health_check_user = (os.getusername() if self._nosetuid_health_checks
else assigned_task.task.job.role)
def wrapper(cmd):
return wrap_with_mesos_containerizer(
cmd,
health_check_user,
sandbox.container_root,
self._mesos_containerizer_path)
shell_signaler = ShellHealthCheck(
cmd=interpolated_command,
preexec_fn=demote_to_job_role_user,
timeout_secs=timeout_secs,
wrapper_fn=wrapper)
a_health_checker = lambda: shell_signaler()
else:
portmap = resolve_ports(mesos_task, assigned_task.assignedPorts)
if 'health' not in portmap:
return None
http_config = health_checker.get(HTTP_HEALTH_CHECK, {})
http_endpoint = http_config.get('endpoint')
http_expected_response = http_config.get('expected_response')
http_expected_response_code = http_config.get('expected_response_code')
http_signaler = HttpSignaler(
portmap['health'],
timeout_secs=timeout_secs)
a_health_checker = lambda: http_signaler(
endpoint=http_endpoint,
expected_response=http_expected_response,
expected_response_code=http_expected_response_code
)
health_checker = HealthChecker(
a_health_checker,
sandbox,
interval_secs=health_check_config.get('interval_secs'),
initial_interval_secs=health_check_config.get('initial_interval_secs'),
max_consecutive_failures=health_check_config.get('max_consecutive_failures'))
return health_checker