def marathon_smartstack_status(
service: str,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
service_namespace_config: ServiceNamespaceConfig,
tasks: Sequence[MarathonTask],
should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist(
system_deploy_blacklist=settings.system_paasta_config.
get_deploy_blacklist())
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist, whitelist=None)
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves, attribute=discover_location_type)
# rebuild the dict, replacing the slave object with just their hostname
slave_hostname_by_location = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance, settings.cluster)
expected_count_per_location = int(expected_smartstack_count /
len(slave_hostname_by_location))
smartstack_status: MutableMapping[str, Any] = {
"registration": registration,
"expected_backends_per_location": expected_count_per_location,
"locations": [],
}
for location, hosts in slave_hostname_by_location.items():
synapse_host = hosts[0]
sorted_backends = sorted(
get_backends(
registration,
synapse_host=synapse_host,
synapse_port=settings.system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=settings.system_paasta_config.
get_synapse_haproxy_url_format(),
),
key=lambda backend: backend["status"],
reverse=True, # put 'UP' backends above 'MAINT' backends
)
matched_backends_and_tasks = match_backends_and_tasks(
sorted_backends, tasks)
location_dict = build_smartstack_location_dict(
location, matched_backends_and_tasks,
should_return_individual_backends)
smartstack_status["locations"].append(location_dict)
return smartstack_status
def status_smartstack_backends(service, instance, job_config, cluster, tasks,
expected_count, soa_dir, verbose, synapse_port,
synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
nerve_ns = marathon_tools.read_namespace_for_service_instance(
service, instance, cluster)
service_instance = compose_job_id(service, nerve_ns)
service_namespace_config = marathon_tools.load_service_namespace_config(
service, instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist, whitelist=[])
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
# rebuild the dict, replacing the slave object
# with just their hostname
grouped_slave_hostname = {
attribute_value: [slave['hostname'] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
if len(grouped_slave_hostname) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" %
service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(
pretty_print_smartstack_backends_for_locations(
service_instance=service_instance,
tasks=tasks,
locations=grouped_slave_hostname,
expected_count=expected_count,
verbose=verbose,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
))
return "\n".join(output)
def status_smartstack_backends(service, instance, job_config, cluster, tasks, expected_count, soa_dir, verbose,
synapse_port, synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
service_instance = marathon_tools.read_registration_for_service_instance(
service, instance, cluster
)
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist,
whitelist=[]
)
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
# rebuild the dict, replacing the slave object
# with just their hostname
grouped_slave_hostname = {
attribute_value: [slave['hostname'] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
if len(grouped_slave_hostname) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" % service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(pretty_print_smartstack_backends_for_locations(
service_instance=service_instance,
tasks=tasks,
locations=grouped_slave_hostname,
expected_count=expected_count,
verbose=verbose,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
))
return "\n".join(output)
def get_smartstack_replication_for_attribute(
attribute: str,
service: str,
namespace: str,
blacklist: DeployBlacklist,
system_paasta_config: SystemPaastaConfig,
) -> Dict[str, Dict[str, int]]:
"""Loads smartstack replication from a host with the specified attribute
:param attribute: a Mesos attribute
:param service: A service name, like 'example_service'
:param namespace: A particular smartstack namespace to inspect, like 'main'
:param constraints: A list of Marathon constraints to restrict which synapse hosts to query
:param blacklist: A list of blacklisted location tuples in the form of (location, value)
:param system_paasta_config: A SystemPaastaConfig object representing the system configuration.
:returns: a dictionary of the form {'<unique_attribute_value>': <smartstack replication hash>}
(the dictionary will contain keys for unique all attribute values)
"""
replication_info = {}
filtered_slaves = mesos_tools.get_all_slaves_for_blacklist_whitelist(
blacklist=blacklist,
whitelist=None,
)
if not filtered_slaves:
raise NoSlavesAvailableError
attribute_slave_dict = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=attribute,
)
full_name = compose_job_id(service, namespace)
for value, hosts in attribute_slave_dict.items():
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]['hostname']
repl_info = get_replication_for_services(
synapse_host=synapse_host,
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.
get_synapse_haproxy_url_format(),
services=[full_name],
)
replication_info[value] = repl_info
return replication_info
def get_smartstack_replication_for_attribute(attribute, service, namespace, blacklist, system_paasta_config):
"""Loads smartstack replication from a host with the specified attribute
:param attribute: a Mesos attribute
:param service: A service name, like 'example_service'
:param namespace: A particular smartstack namespace to inspect, like 'main'
:param constraints: A list of Marathon constraints to restrict which synapse hosts to query
:param blacklist: A list of blacklisted location tuples in the form of (location, value)
:param system_paasta_config: A SystemPaastaConfig object representing the system configuration.
:returns: a dictionary of the form {'<unique_attribute_value>': <smartstack replication hash>}
(the dictionary will contain keys for unique all attribute values)
"""
replication_info = {}
filtered_slaves = mesos_tools.get_all_slaves_for_blacklist_whitelist(
blacklist=blacklist,
whitelist=[],
)
if not filtered_slaves:
raise mesos_tools.NoSlavesAvailableError
attribute_slave_dict = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=attribute
)
full_name = compose_job_id(service, namespace)
for value, hosts in attribute_slave_dict.iteritems():
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]['hostname']
repl_info = get_replication_for_services(
synapse_host=synapse_host,
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.get_synapse_haproxy_url_format(),
services=[full_name],
)
replication_info[value] = repl_info
return replication_info
