def pretty_print_smartstack_backends_for_locations(service_instance, tasks, locations, expected_count, verbose,
synapse_port, synapse_haproxy_url_format):
"""
Pretty prints the status of smartstack backends of a specified service and instance in the specified locations
"""
rows = [(" Name", "LastCheck", "LastChange", "Status")] if verbose else []
expected_count_per_location = int(expected_count / len(locations))
for location in sorted(locations):
hosts = locations[location]
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]
sorted_backends = sorted(
get_backends(
service_instance,
synapse_host=synapse_host,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
),
key=lambda backend: backend['status'],
reverse=True, # Specify reverse so that backends in 'UP' are placed above 'MAINT'
)
matched_tasks = match_backends_and_tasks(sorted_backends, tasks)
running_count = sum(1 for backend, task in matched_tasks if backend and backend_is_up(backend))
rows.append(" %s - %s" % (location, haproxy_backend_report(expected_count_per_location, running_count)))
# If verbose mode is specified, show status of individual backends
if verbose:
for backend, task in matched_tasks:
if backend is not None:
rows.append(format_haproxy_backend_row(backend, task is not None))
return format_table(rows)
def pretty_print_smartstack_backends_for_locations(service_instance, tasks, locations, expected_count, verbose,
synapse_port, synapse_haproxy_url_format):
"""
Pretty prints the status of smartstack backends of a specified service and instance in the specified locations
"""
rows = [(" Name", "LastCheck", "LastChange", "Status")] if verbose else []
expected_count_per_location = int(expected_count / len(locations))
for location in sorted(locations):
hosts = locations[location]
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]
sorted_backends = sorted(
get_backends(
service_instance,
synapse_host=synapse_host,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
),
key=lambda backend: backend['status'],
reverse=True, # Specify reverse so that backends in 'UP' are placed above 'MAINT'
)
matched_tasks = match_backends_and_tasks(sorted_backends, tasks)
running_count = sum(1 for backend, task in matched_tasks if backend and backend_is_up(backend))
rows.append(" %s - %s" % (location, haproxy_backend_report(expected_count_per_location, running_count)))
# If verbose mode is specified, show status of individual backends
if verbose:
for backend, task in matched_tasks:
if backend is not None:
rows.append(format_haproxy_backend_row(backend, task is not None))
return format_table(rows)
def _build_smartstack_location_dict_for_backends(
synapse_host: str,
registration: str,
tasks: Sequence[MarathonTask],
location: str,
should_return_individual_backends: bool,
) -> MutableMapping[str, Any]:
sorted_smartstack_backends = sorted(
smartstack_tools.get_backends(
registration,
synapse_host=synapse_host,
synapse_port=settings.system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=settings.system_paasta_config.
get_synapse_haproxy_url_format(),
),
key=lambda backend: backend["status"],
reverse=True, # put 'UP' backends above 'MAINT' backends
)
matched_smartstack_backends_and_tasks = smartstack_tools.match_backends_and_tasks(
sorted_smartstack_backends, tasks)
return smartstack_tools.build_smartstack_location_dict(
location,
matched_smartstack_backends_and_tasks,
should_return_individual_backends,
)
def test_match_backends_and_tasks():
backends = [
{
"pxname": "servicename.main",
"svname": "10.50.2.4:31000_box4",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.5:31001_box5",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.6:31001_box6",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.6:31002_box7",
"status": "UP"
},
{
"pxname": "servicename.main",
"svname": "10.50.2.8:31000_box8",
"status": "UP"
},
]
hostnames = {
'box4': '10.50.2.4',
'box5': '10.50.2.5',
'box6': '10.50.2.6',
'box7': '10.50.2.7',
'box8': '10.50.2.8',
}
good_task1 = mock.Mock(host='box4', ports=[31000])
good_task2 = mock.Mock(host='box5', ports=[31001])
bad_task = mock.Mock(host='box7', ports=[31000])
tasks = [good_task1, good_task2, bad_task]
with mock.patch(
'paasta_tools.smartstack_tools.socket.gethostbyname',
side_effect=lambda x: hostnames[x],
autospec=True,
):
expected = [
(backends[0], good_task1),
(backends[1], good_task2),
(None, bad_task),
(backends[2], None),
(backends[3], None),
(backends[4], None),
]
actual = match_backends_and_tasks(backends, tasks)
assert sorted(actual) == sorted(expected)
def marathon_smartstack_status(
service: str,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
service_namespace_config: ServiceNamespaceConfig,
tasks: Sequence[MarathonTask],
should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist(
system_deploy_blacklist=settings.system_paasta_config.
get_deploy_blacklist())
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist, whitelist=None)
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves, attribute=discover_location_type)
# rebuild the dict, replacing the slave object with just their hostname
slave_hostname_by_location = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance, settings.cluster)
expected_count_per_location = int(expected_smartstack_count /
len(slave_hostname_by_location))
smartstack_status: MutableMapping[str, Any] = {
"registration": registration,
"expected_backends_per_location": expected_count_per_location,
"locations": [],
}
for location, hosts in slave_hostname_by_location.items():
synapse_host = hosts[0]
sorted_backends = sorted(
get_backends(
registration,
synapse_host=synapse_host,
synapse_port=settings.system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=settings.system_paasta_config.
get_synapse_haproxy_url_format(),
),
key=lambda backend: backend["status"],
reverse=True, # put 'UP' backends above 'MAINT' backends
)
matched_backends_and_tasks = match_backends_and_tasks(
sorted_backends, tasks)
location_dict = build_smartstack_location_dict(
location, matched_backends_and_tasks,
should_return_individual_backends)
smartstack_status["locations"].append(location_dict)
return smartstack_status
def test_match_backends_and_tasks():
backends = [
{"pxname": "servicename.main", "svname": "10.50.2.4:31000_box4", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.5:31001_box5", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31001_box6", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.6:31002_box7", "status": "UP"},
{"pxname": "servicename.main", "svname": "10.50.2.8:31000_box8", "status": "UP"},
]
hostnames = {
'box4': '10.50.2.4',
'box5': '10.50.2.5',
'box6': '10.50.2.6',
'box7': '10.50.2.7',
'box8': '10.50.2.8',
}
good_task1 = mock.Mock(host='box4', ports=[31000])
good_task2 = mock.Mock(host='box5', ports=[31001])
bad_task = mock.Mock(host='box7', ports=[31000])
tasks = [good_task1, good_task2, bad_task]
with mock.patch(
'paasta_tools.smartstack_tools.socket.gethostbyname',
side_effect=lambda x: hostnames[x],
autospec=True,
):
expected = [
(backends[0], good_task1),
(backends[1], good_task2),
(None, bad_task),
(backends[2], None),
(backends[3], None),
(backends[4], None),
]
actual = match_backends_and_tasks(backends, tasks)
assert sorted(actual) == sorted(expected)
def test_match_backends_and_tasks():
backends = [
{
"pxname": "servicename.main",
"svname": "10.50.2.4:31000_box4",
"status": "UP",
},
{
"pxname": "servicename.main",
"svname": "10.50.2.5:31001_box5",
"status": "UP",
},
{
"pxname": "servicename.main",
"svname": "10.50.2.6:31001_box6",
"status": "UP",
},
{
"pxname": "servicename.main",
"svname": "10.50.2.6:31002_box7",
"status": "UP",
},
{
"pxname": "servicename.main",
"svname": "10.50.2.8:31000_box8",
"status": "UP",
},
]
hostnames = {
"box4": "10.50.2.4",
"box5": "10.50.2.5",
"box6": "10.50.2.6",
"box7": "10.50.2.7",
"box8": "10.50.2.8",
}
good_task1 = mock.Mock(host="box4", ports=[31000])
good_task2 = mock.Mock(host="box5", ports=[31001])
bad_task = mock.Mock(host="box7", ports=[31000])
tasks = [good_task1, good_task2, bad_task]
with mock.patch(
"paasta_tools.smartstack_tools.socket.gethostbyname",
side_effect=lambda x: hostnames[x],
autospec=True,
):
expected = [
(backends[0], good_task1),
(backends[1], good_task2),
(None, bad_task),
(backends[2], None),
(backends[3], None),
(backends[4], None),
]
actual = match_backends_and_tasks(backends, tasks)
def keyfunc(t):
return tuple(sorted((t[0] or {}).items())), t[1]
assert sorted(actual, key=keyfunc) == sorted(expected, key=keyfunc)
