def get_happy_tasks(app,
service,
nerve_ns,
system_paasta_config,
min_task_uptime=None,
check_haproxy=False):
"""Given a MarathonApp object, return the subset of tasks which are considered healthy.
With the default options, this returns tasks where at least one of the defined Marathon healthchecks passes.
For it to do anything interesting, set min_task_uptime or check_haproxy.
:param app: A MarathonApp object.
:param service: The name of the service.
:param nerve_ns: The nerve namespace
:param min_task_uptime: Minimum number of seconds that a task must be running before we consider it healthy. Useful
if tasks take a while to start up.
:param check_haproxy: Whether to check the local haproxy to make sure this task has been registered and discovered.
"""
tasks = app.tasks
happy = []
now = datetime.datetime.utcnow()
if check_haproxy:
tasks_in_smartstack = []
service_namespace = compose_job_id(service, nerve_ns)
service_namespace_config = marathon_tools.load_service_namespace_config(
service, nerve_ns)
discover_location_type = service_namespace_config.get_discover()
unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(
discover_location_type)
for value, hosts in unique_values.iteritems():
synapse_host = hosts[0]
tasks_in_smartstack.extend(
get_registered_marathon_tasks(
synapse_host,
system_paasta_config.get_synapse_port(),
system_paasta_config.get_synapse_haproxy_url_format(),
service_namespace,
tasks,
))
tasks = tasks_in_smartstack
for task in tasks:
if min_task_uptime is not None:
if (now - task.started_at).total_seconds() < min_task_uptime:
continue
# if there are healthchecks defined for the app but none have executed yet, then task is unhappy
if len(app.health_checks) > 0 and len(task.health_check_results) == 0:
continue
# if there are health check results, check if at least one healthcheck is passing
if not marathon_tools.is_task_healthy(
task, require_all=False, default_healthy=True):
continue
happy.append(task)
return happy
def get_happy_tasks(app, service, nerve_ns, system_paasta_config, min_task_uptime=None, check_haproxy=False):
"""Given a MarathonApp object, return the subset of tasks which are considered healthy.
With the default options, this returns tasks where at least one of the defined Marathon healthchecks passes.
For it to do anything interesting, set min_task_uptime or check_haproxy.
:param app: A MarathonApp object.
:param service: The name of the service.
:param nerve_ns: The nerve namespace
:param min_task_uptime: Minimum number of seconds that a task must be running before we consider it healthy. Useful
if tasks take a while to start up.
:param check_haproxy: Whether to check the local haproxy to make sure this task has been registered and discovered.
"""
tasks = app.tasks
happy = []
now = datetime.datetime.utcnow()
if check_haproxy:
tasks_in_smartstack = []
service_namespace = compose_job_id(service, nerve_ns)
service_namespace_config = marathon_tools.load_service_namespace_config(service, nerve_ns)
discover_location_type = service_namespace_config.get_discover()
unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(discover_location_type)
for value, hosts in unique_values.iteritems():
synapse_host = hosts[0]
tasks_in_smartstack.extend(get_registered_marathon_tasks(
synapse_host,
system_paasta_config.get_synapse_port(),
system_paasta_config.get_synapse_haproxy_url_format(),
service_namespace,
tasks,
))
tasks = tasks_in_smartstack
for task in tasks:
if task.started_at is None:
# Can't be healthy if it hasn't started
continue
if min_task_uptime is not None:
if (now - task.started_at).total_seconds() < min_task_uptime:
continue
# if there are healthchecks defined for the app but none have executed yet, then task is unhappy
if len(app.health_checks) > 0 and len(task.health_check_results) == 0:
continue
# if there are health check results, check if at least one healthcheck is passing
if not marathon_tools.is_task_healthy(task, require_all=False, default_healthy=True):
continue
happy.append(task)
return happy
def test_get_registered_marathon_tasks():
backends = [
{"pxname": "servicename.main", "svname": "10.50.2.4:31000_box4", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.5:31001_box5", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31001_box6", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31002_box7", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.8:31000_box8", "status": "UP"},
]
hostnames = {
'box4': '10.50.2.4',
'box5': '10.50.2.5',
'box6': '10.50.2.6',
'box7': '10.50.2.7',
'box8': '10.50.2.8',
}
good_task1 = mock.Mock(host='box4', ports=[31000])
good_task2 = mock.Mock(host='box5', ports=[31001])
bad_task = mock.Mock(host='box7', ports=[31000])
marathon_tasks = [
good_task1,
good_task2,
bad_task,
]
with mock.patch(
'paasta_tools.monitoring.replication_utils.get_multiple_backends',
return_value=backends
) as mock_get_multiple_backends:
with mock.patch(
'paasta_tools.monitoring.replication_utils.'
'socket.gethostbyname',
side_effect=lambda x: hostnames[x],
):
actual = get_registered_marathon_tasks(
'fake_host',
6666,
DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
'servicename.main',
marathon_tasks,
)
expected = [good_task1, good_task2]
assert actual == expected
mock_get_multiple_backends.assert_called_once_with(
['servicename.main'],
synapse_host='fake_host',
synapse_port=6666,
synapse_haproxy_url_format=DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
)
def test_get_registered_marathon_tasks():
backends = [
{"pxname": "servicename.main", "svname": "10.50.2.4:31000_box4", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.5:31001_box5", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31001_box6", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31002_box7", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.8:31000_box8", "status": "UP"},
]
hostnames = {
'box4': '10.50.2.4',
'box5': '10.50.2.5',
'box6': '10.50.2.6',
'box7': '10.50.2.7',
'box8': '10.50.2.8',
}
good_task1 = mock.Mock(host='box4', ports=[31000])
good_task2 = mock.Mock(host='box5', ports=[31001])
bad_task = mock.Mock(host='box7', ports=[31000])
marathon_tasks = [
good_task1,
good_task2,
bad_task,
]
with mock.patch(
'paasta_tools.monitoring.replication_utils.get_multiple_backends',
return_value=backends
) as mock_get_multiple_backends:
with mock.patch(
'paasta_tools.monitoring.replication_utils.'
'socket.gethostbyname',
side_effect=lambda x: hostnames[x],
):
actual = get_registered_marathon_tasks(
'fake_host',
6666,
DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
'servicename.main',
marathon_tasks,
)
expected = [good_task1, good_task2]
assert actual == expected
mock_get_multiple_backends.assert_called_once_with(
['servicename.main'],
synapse_host='fake_host',
synapse_port=6666,
synapse_haproxy_url_format=DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT,
)
def test_get_registered_marathon_tasks():
backends = [
{"pxname": "servicename.main", "svname": "10.50.2.4:31000_box4", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.5:31001_box5", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31001_box6", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31002_box7", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.8:31000_box8", "status": "UP"},
]
hostnames = {
'box4': '10.50.2.4',
'box5': '10.50.2.5',
'box6': '10.50.2.6',
'box7': '10.50.2.7',
'box8': '10.50.2.8',
}
good_task1 = mock.Mock(host='box4', ports=[31000])
good_task2 = mock.Mock(host='box5', ports=[31001])
bad_task = mock.Mock(host='box7', ports=[31000])
marathon_tasks = [
good_task1,
good_task2,
bad_task,
]
with mock.patch(
'paasta_tools.monitoring.replication_utils.get_multiple_backends',
return_value=backends
):
with mock.patch(
'paasta_tools.monitoring.replication_utils.'
'socket.gethostbyname',
side_effect=lambda x: hostnames[x],
):
actual = get_registered_marathon_tasks(
'fake_host',
6666,
'servicename.main',
marathon_tasks,
)
expected = [good_task1, good_task2]
assert actual == expected
def test_get_registered_marathon_tasks():
backends = [
{
"pxname": "servicename.main",
"svname": "10.50.2.4:31000_box4",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.5:31001_box5",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.6:31001_box6",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.6:31002_box7",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.8:31000_box8",
"status": "UP"
},
]
hostnames = {
'box4': '10.50.2.4',
'box5': '10.50.2.5',
'box6': '10.50.2.6',
'box7': '10.50.2.7',
'box8': '10.50.2.8',
}
good_task1 = mock.Mock(host='box4', ports=[31000])
good_task2 = mock.Mock(host='box5', ports=[31001])
bad_task = mock.Mock(host='box7', ports=[31000])
marathon_tasks = [
good_task1,
good_task2,
bad_task,
]
with mock.patch(
'paasta_tools.monitoring.replication_utils.get_multiple_backends',
return_value=backends):
with mock.patch(
'paasta_tools.monitoring.replication_utils.'
'socket.gethostbyname',
side_effect=lambda x: hostnames[x],
):
actual = get_registered_marathon_tasks(
'fake_host',
6666,
'servicename.main',
marathon_tasks,
)
expected = [good_task1, good_task2]
assert actual == expected
