def test_get_mesos_slaves_grouped_by_attribute_uses_blacklist(
mock_filter_mesos_slaves_by_blacklist,
mock_fetch_state
):
fake_blacklist = ['fake_blacklist']
fake_whitelist = []
fake_slaves = [
{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': 'fake_value_1',
}
},
{
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': 'fake_value_1',
}
}
]
mock_fetch_state.return_value = {'slaves': fake_slaves}
mock_filter_mesos_slaves_by_blacklist.return_value = fake_slaves
mesos_tools.get_mesos_slaves_grouped_by_attribute('fake_attribute', blacklist=fake_blacklist)
mock_filter_mesos_slaves_by_blacklist.assert_called_once_with(slaves=fake_slaves, blacklist=fake_blacklist,
whitelist=fake_whitelist)
def test_get_mesos_slaves_grouped_by_attribute_uses_blacklist(mock_filter_mesos_slaves_by_blacklist, mock_fetch_state):
fake_blacklist = ["fake_blacklist"]
fake_slaves = [
{"hostname": "fake_host_1", "attributes": {"fake_attribute": "fake_value_1"}},
{"hostname": "fake_host_2", "attributes": {"fake_attribute": "fake_value_1"}},
]
mock_fetch_state.return_value = {"slaves": fake_slaves}
mock_filter_mesos_slaves_by_blacklist.return_value = fake_slaves
mesos_tools.get_mesos_slaves_grouped_by_attribute("fake_attribute", blacklist=fake_blacklist)
mock_filter_mesos_slaves_by_blacklist.assert_called_once_with(slaves=fake_slaves, blacklist=fake_blacklist)
def _get_allowed_locations_and_hosts(
self, instance_config: InstanceConfig
) -> Dict[str, Sequence[SmartstackHost]]:
"""Returns a dict of locations and lists of corresponding mesos slaves
where deployment of the instance is allowed.
:param instance_config: An instance of MarathonServiceConfig
:returns: A dict {"uswest1-prod": [SmartstackHost(), SmartstackHost(), ...]}
"""
monitoring_blacklist = instance_config.get_monitoring_blacklist(
system_deploy_blacklist=self._system_paasta_config.
get_deploy_blacklist(), )
filtered_slaves = mesos_tools.filter_mesos_slaves_by_blacklist(
slaves=self._mesos_slaves,
blacklist=monitoring_blacklist,
whitelist=None,
)
discover_location_type = marathon_tools.load_service_namespace_config(
service=instance_config.service,
namespace=instance_config.instance,
soa_dir=instance_config.soa_dir,
).get_discover()
attribute_to_slaves = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
ret: Dict[str, Sequence[SmartstackHost]] = {}
for attr, slaves in attribute_to_slaves.items():
ret[attr] = [
SmartstackHost(hostname=slave['hostname'],
pool=slave['attributes']['pool'])
for slave in slaves
]
return ret
def get_routing_constraints(self, service_namespace_config):
discover_level = service_namespace_config.get_discover()
locations = get_mesos_slaves_grouped_by_attribute(
attribute=discover_level, blacklist=self.get_deploy_blacklist())
routing_constraints = [[discover_level, "GROUP_BY", str(len(locations))]]
return routing_constraints
def test_get_mesos_slaves_grouped_by_attribute(mock_fetch_state):
fake_attribute = 'fake_attribute'
fake_value_1 = 'fake_value_1'
fake_value_2 = 'fake_value_2'
mock_fetch_state.return_value = {
'slaves': [{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': fake_value_1,
}
}, {
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': fake_value_2,
}
}, {
'hostname': 'fake_host_3',
'attributes': {
'fake_attribute': fake_value_1,
}
}, {
'hostname': 'fake_host_4',
'attributes': {
'fake_attribute': 'fake_other_value',
}
}]
}
expected = {
'fake_value_1': ['fake_host_1', 'fake_host_3'],
'fake_value_2': ['fake_host_2'],
'fake_other_value': ['fake_host_4'],
}
actual = mesos_tools.get_mesos_slaves_grouped_by_attribute(fake_attribute)
assert actual == expected
def get_allowed_locations_and_hosts(
self, instance_config: LongRunningServiceConfig
) -> Dict[str, Sequence[DiscoveredHost]]:
"""Returns a dict of locations and lists of corresponding mesos slaves
where deployment of the instance is allowed.
:param instance_config: An instance of MarathonServiceConfig
:returns: A dict {"uswest1-prod": [DiscoveredHost(), DiscoveredHost(), ...]}
"""
discover_location_type = marathon_tools.load_service_namespace_config(
service=instance_config.service,
namespace=instance_config.get_nerve_namespace(),
soa_dir=instance_config.soa_dir,
).get_discover()
attribute_to_slaves = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=self._mesos_slaves, attribute=discover_location_type
)
ret: Dict[str, Sequence[DiscoveredHost]] = {}
for attr, slaves in attribute_to_slaves.items():
ret[attr] = [
DiscoveredHost(
hostname=slave["hostname"], pool=slave["attributes"]["pool"]
)
for slave in slaves
]
return ret
def test_get_mesos_slaves_grouped_by_attribute():
fake_value_1 = "fake_value_1"
fake_value_2 = "fake_value_2"
fake_slaves = [
{
"hostname": "fake_host_1",
"attributes": {
"fake_attribute": fake_value_1
}
},
{
"hostname": "fake_host_2",
"attributes": {
"fake_attribute": fake_value_2
}
},
{
"hostname": "fake_host_3",
"attributes": {
"fake_attribute": fake_value_1
}
},
{
"hostname": "fake_host_4",
"attributes": {
"fake_attribute": "fake_other_value"
},
},
]
expected = {
"fake_value_1": [
{
"hostname": "fake_host_1",
"attributes": {
"fake_attribute": fake_value_1
}
},
{
"hostname": "fake_host_3",
"attributes": {
"fake_attribute": fake_value_1
}
},
],
"fake_value_2": [{
"hostname": "fake_host_2",
"attributes": {
"fake_attribute": fake_value_2
}
}],
"fake_other_value": [{
"hostname": "fake_host_4",
"attributes": {
"fake_attribute": "fake_other_value"
},
}],
}
actual = mesos_tools.get_mesos_slaves_grouped_by_attribute(
fake_slaves, "fake_attribute")
assert actual == expected
def get_constraints(self, service_namespace_config):
"""Gets the constraints specified in the service's marathon configuration.
These are Marathon app constraints. See
https://mesosphere.github.io/marathon/docs/constraints.html
Defaults to `GROUP_BY region`. If the service's smartstack configuration
specifies a `discover` key, then defaults to `GROUP_BY <value of discover>` instead.
:param service_namespace_config: The service instance's configuration dictionary
:returns: The constraints specified in the config, or defaults described above
"""
if 'constraints' in self.config_dict:
return self.config_dict.get('constraints')
else:
discover_level = service_namespace_config.get_discover()
locations = get_mesos_slaves_grouped_by_attribute(
attribute=discover_level,
blacklist=self.get_deploy_blacklist())
deploy_constraints = deploy_blacklist_to_constraints(
self.get_deploy_blacklist())
routing_constraints = [[
discover_level, "GROUP_BY",
str(len(locations))
]]
return routing_constraints + deploy_constraints
def _get_allowed_locations_and_hostnames(
self, instance_config) -> Dict[str, list]:
"""Returns a dict of locations and lists of corresponding mesos slaves
where deployment of the instance is allowed.
:param instance_config: An instance of MarathonServiceConfig
:returns: A dict {"uswest1-prod": ['hostname1', 'hostname2], ...}.
"""
monitoring_blacklist = instance_config.get_monitoring_blacklist(
system_deploy_blacklist=self._system_paasta_config.
get_deploy_blacklist(), )
filtered_slaves = mesos_tools.filter_mesos_slaves_by_blacklist(
slaves=self._mesos_slaves,
blacklist=monitoring_blacklist,
whitelist=None,
)
discover_location_type = marathon_tools.load_service_namespace_config(
service=instance_config.service,
namespace=instance_config.instance,
soa_dir=instance_config.soa_dir,
).get_discover()
slaves_grouped_by_attribute = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
return {
attr: [s['hostname'] for s in slaves]
for attr, slaves in slaves_grouped_by_attribute.items()
}
def get_smartstack_replication_for_attribute(attribute, service, namespace, blacklist):
"""Loads smartstack replication from a host with the specified attribute
:param attribute: a Mesos attribute
:param service: A service name, like 'example_service'
:param namespace: A particular smartstack namespace to inspect, like 'main'
:param constraints: A list of Marathon constraints to restrict which synapse hosts to query
:param blacklist: A list of blacklisted location tuples in the form of (location, value)
:returns: a dictionary of the form {'<unique_attribute_value>': <smartstack replication hash>}
(the dictionary will contain keys for unique all attribute values)
"""
replication_info = {}
unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(attribute=attribute, blacklist=blacklist)
full_name = compose_job_id(service, namespace)
for value, hosts in unique_values.iteritems():
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]
repl_info = replication_utils.get_replication_for_services(
synapse_host=synapse_host,
synapse_port=smartstack_tools.DEFAULT_SYNAPSE_PORT,
services=[full_name],
)
replication_info[value] = repl_info
return replication_info
def get_routing_constraints(self, service_namespace_config):
discover_level = service_namespace_config.get_discover()
locations = get_mesos_slaves_grouped_by_attribute(
attribute=discover_level, blacklist=self.get_deploy_blacklist(),
whitelist=self.get_deploy_whitelist())
routing_constraints = [[discover_level, "GROUP_BY", str(len(locations))]]
return routing_constraints
def get_happy_tasks(app, service, nerve_ns, system_paasta_config, min_task_uptime=None, check_haproxy=False):
"""Given a MarathonApp object, return the subset of tasks which are considered healthy.
With the default options, this returns tasks where at least one of the defined Marathon healthchecks passes.
For it to do anything interesting, set min_task_uptime or check_haproxy.
:param app: A MarathonApp object.
:param service: The name of the service.
:param nerve_ns: The nerve namespace
:param min_task_uptime: Minimum number of seconds that a task must be running before we consider it healthy. Useful
if tasks take a while to start up.
:param check_haproxy: Whether to check the local haproxy to make sure this task has been registered and discovered.
"""
tasks = app.tasks
happy = []
now = datetime.datetime.utcnow()
if check_haproxy:
tasks_in_smartstack = []
service_namespace = compose_job_id(service, nerve_ns)
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=nerve_ns)
discover_location_type = service_namespace_config.get_discover()
unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=mesos_tools.get_slaves(),
attribute=discover_location_type
)
for value, hosts in unique_values.iteritems():
synapse_hostname = hosts[0]['hostname']
tasks_in_smartstack.extend(get_registered_marathon_tasks(
synapse_hostname,
system_paasta_config.get_synapse_port(),
system_paasta_config.get_synapse_haproxy_url_format(),
service_namespace,
tasks,
))
tasks = tasks_in_smartstack
for task in tasks:
if task.started_at is None:
# Can't be healthy if it hasn't started
continue
if min_task_uptime is not None:
if (now - task.started_at).total_seconds() < min_task_uptime:
continue
# if there are healthchecks defined for the app but none have executed yet, then task is unhappy
if len(app.health_checks) > 0 and len(task.health_check_results) == 0:
continue
# if there are health check results, check if at least one healthcheck is passing
if not marathon_tools.is_task_healthy(task, require_all=False, default_healthy=True):
continue
happy.append(task)
return happy
def status_smartstack_backends(
service,
instance,
job_config,
service_namespace_config,
cluster,
tasks,
expected_count,
soa_dir,
synapse_port,
synapse_haproxy_url_format,
system_deploy_blacklist,
system_deploy_whitelist,
verbose,
):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=get_slaves(), attribute=discover_location_type
)
# rebuild the dict, replacing the slave object
# with just their hostname
grouped_slave_hostname = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
if len(grouped_slave_hostname) == 0:
output.append(
"Smartstack: ERROR - %s is NOT in smartstack at all!" % registration
)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % registration)
output.append(" Backends:")
output.extend(
pretty_print_smartstack_backends_for_locations(
registration=registration,
tasks=tasks,
locations=grouped_slave_hostname,
expected_count=expected_count,
verbose=verbose,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
)
)
return "\n".join(output)
def test_get_mesos_slaves_grouped_by_attribute():
fake_value_1 = 'fake_value_1'
fake_value_2 = 'fake_value_2'
fake_slaves = [{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': fake_value_1,
}
}, {
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': fake_value_2,
}
}, {
'hostname': 'fake_host_3',
'attributes': {
'fake_attribute': fake_value_1,
}
}, {
'hostname': 'fake_host_4',
'attributes': {
'fake_attribute': 'fake_other_value',
}
}]
expected = {
'fake_value_1': [
{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': fake_value_1,
}
},
{
'hostname': 'fake_host_3',
'attributes': {
'fake_attribute': fake_value_1,
}
},
],
'fake_value_2': [
{
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': fake_value_2,
}
},
],
'fake_other_value': [{
'hostname': 'fake_host_4',
'attributes': {
'fake_attribute': 'fake_other_value',
}
}]
}
actual = mesos_tools.get_mesos_slaves_grouped_by_attribute(
fake_slaves, 'fake_attribute')
assert actual == expected
def marathon_smartstack_status(
service: str,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
service_namespace_config: ServiceNamespaceConfig,
tasks: Sequence[MarathonTask],
should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist(
system_deploy_blacklist=settings.system_paasta_config.
get_deploy_blacklist())
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist, whitelist=None)
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves, attribute=discover_location_type)
# rebuild the dict, replacing the slave object with just their hostname
slave_hostname_by_location = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance, settings.cluster)
expected_count_per_location = int(expected_smartstack_count /
len(slave_hostname_by_location))
smartstack_status: MutableMapping[str, Any] = {
"registration": registration,
"expected_backends_per_location": expected_count_per_location,
"locations": [],
}
for location, hosts in slave_hostname_by_location.items():
synapse_host = hosts[0]
sorted_backends = sorted(
get_backends(
registration,
synapse_host=synapse_host,
synapse_port=settings.system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=settings.system_paasta_config.
get_synapse_haproxy_url_format(),
),
key=lambda backend: backend["status"],
reverse=True, # put 'UP' backends above 'MAINT' backends
)
matched_backends_and_tasks = match_backends_and_tasks(
sorted_backends, tasks)
location_dict = build_smartstack_location_dict(
location, matched_backends_and_tasks,
should_return_individual_backends)
smartstack_status["locations"].append(location_dict)
return smartstack_status
def marathon_service_mesh_status(
service: str,
service_mesh: pik.ServiceMesh,
instance: str,
job_config: marathon_tools.MarathonServiceConfig,
service_namespace_config: ServiceNamespaceConfig,
tasks: Sequence[MarathonTask],
should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
registration = job_config.get_registrations()[0]
discover_location_type = service_namespace_config.get_discover()
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=get_slaves(), attribute=discover_location_type)
# rebuild the dict, replacing the slave object with just their hostname
slave_hostname_by_location = {
attribute_value: [slave["hostname"] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
expected_instance_count = marathon_tools.get_expected_instance_count_for_namespace(
service, instance, settings.cluster)
expected_count_per_location = int(expected_instance_count /
len(slave_hostname_by_location))
service_mesh_status: MutableMapping[str, Any] = {
"registration": registration,
"expected_backends_per_location": expected_count_per_location,
"locations": [],
}
for location, hosts in slave_hostname_by_location.items():
if service_mesh == pik.ServiceMesh.SMARTSTACK:
service_mesh_status["locations"].append(
_build_smartstack_location_dict_for_backends(
synapse_host=hosts[0],
registration=registration,
tasks=tasks,
location=location,
should_return_individual_backends=
should_return_individual_backends,
))
elif service_mesh == pik.ServiceMesh.ENVOY:
service_mesh_status["locations"].append(
_build_envoy_location_dict_for_backends(
envoy_host=hosts[0],
registration=registration,
tasks=tasks,
location=location,
should_return_individual_backends=
should_return_individual_backends,
))
return service_mesh_status
def status_smartstack_backends(service, instance, job_config, cluster, tasks,
expected_count, soa_dir, verbose, synapse_port,
synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
nerve_ns = marathon_tools.read_namespace_for_service_instance(
service, instance, cluster)
service_instance = compose_job_id(service, nerve_ns)
service_namespace_config = marathon_tools.load_service_namespace_config(
service, instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist, whitelist=[])
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
# rebuild the dict, replacing the slave object
# with just their hostname
grouped_slave_hostname = {
attribute_value: [slave['hostname'] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
if len(grouped_slave_hostname) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" %
service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(
pretty_print_smartstack_backends_for_locations(
service_instance=service_instance,
tasks=tasks,
locations=grouped_slave_hostname,
expected_count=expected_count,
verbose=verbose,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
))
return "\n".join(output)
def status_smartstack_backends(service, instance, job_config, cluster, tasks, expected_count, soa_dir, verbose,
synapse_port, synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
service_instance = marathon_tools.read_registration_for_service_instance(
service, instance, cluster
)
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
filtered_slaves = get_all_slaves_for_blacklist_whitelist(
blacklist=monitoring_blacklist,
whitelist=[]
)
grouped_slaves = get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
# rebuild the dict, replacing the slave object
# with just their hostname
grouped_slave_hostname = {
attribute_value: [slave['hostname'] for slave in slaves]
for attribute_value, slaves in grouped_slaves.items()
}
if len(grouped_slave_hostname) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" % service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(pretty_print_smartstack_backends_for_locations(
service_instance=service_instance,
tasks=tasks,
locations=grouped_slave_hostname,
expected_count=expected_count,
verbose=verbose,
synapse_port=synapse_port,
synapse_haproxy_url_format=synapse_haproxy_url_format,
))
return "\n".join(output)
def get_routing_constraints(self, service_namespace_config,
system_paasta_config):
"""
Returns a set of constraints in order to evenly group a marathon
application amongst instances of a discovery type.
If, for example, a given app's 'discover' key is set to 'region', then this function
computes the constraints required to group the app evenly amongst each
of the actual 'region' values in the cluster.
It does so by querying the value of the discover attribute for each expected slave in the cluster (as defined
by the expected_slave_attributes key in system paasta config), returning a GROUP_BY constraint where the value
is the number of unique values for that attribute.
If you have not set expected_slave_attributes in the system paasta config, this function returns an empty list.
:param service_namespace_config: the config for this service
:returns: a set of constraints for marathon
"""
discover_level = service_namespace_config.get_discover()
expected_slave_attributes = system_paasta_config.get_expected_slave_attributes(
)
if expected_slave_attributes is None:
return []
fake_slaves = [{"attributes": a} for a in expected_slave_attributes]
filtered_slaves = filter_mesos_slaves_by_blacklist(
slaves=fake_slaves,
blacklist=self.get_deploy_blacklist(
system_deploy_blacklist=system_paasta_config.
get_deploy_blacklist(), ),
whitelist=self.get_deploy_whitelist(
system_deploy_whitelist=system_paasta_config.
get_deploy_whitelist(), ),
)
if not filtered_slaves:
raise NoSlavesAvailableError((
"We do not believe any slaves on the cluster will match the constraints for %s.%s. If you believe "
"this is incorrect, have your system administrator adjust the value of expected_slave_attributes "
"in the system paasta configs.") %
(self.service, self.instance), )
value_dict = get_mesos_slaves_grouped_by_attribute(
filtered_slaves,
discover_level,
)
routing_constraints = [[
discover_level, "GROUP_BY",
str(len(value_dict.keys()))
]]
return routing_constraints
def status_smartstack_backends(service, instance, job_config, cluster, tasks,
expected_count, soa_dir, verbose, synapse_port,
synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
nerve_ns = marathon_tools.read_namespace_for_service_instance(
service, instance, cluster)
service_instance = compose_job_id(service, nerve_ns)
if instance != nerve_ns:
ns_string = PaastaColors.bold(nerve_ns)
output.append(
"Smartstack: N/A - %s is announced in the %s namespace." %
(instance, ns_string))
# If verbose mode is specified, then continue to show backends anyway, otherwise stop early
if not verbose:
return "\n".join(output)
service_namespace_config = marathon_tools.load_service_namespace_config(
service, instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
unique_attributes = get_mesos_slaves_grouped_by_attribute(
attribute=discover_location_type, blacklist=monitoring_blacklist)
if len(unique_attributes) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" %
service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(
pretty_print_smartstack_backends_for_locations(
service_instance,
tasks,
unique_attributes,
expected_count,
verbose,
synapse_port,
synapse_haproxy_url_format,
))
return "\n".join(output)
def get_smartstack_replication_for_attribute(
attribute: str,
service: str,
namespace: str,
blacklist: DeployBlacklist,
system_paasta_config: SystemPaastaConfig,
) -> Dict[str, Dict[str, int]]:
"""Loads smartstack replication from a host with the specified attribute
:param attribute: a Mesos attribute
:param service: A service name, like 'example_service'
:param namespace: A particular smartstack namespace to inspect, like 'main'
:param constraints: A list of Marathon constraints to restrict which synapse hosts to query
:param blacklist: A list of blacklisted location tuples in the form of (location, value)
:param system_paasta_config: A SystemPaastaConfig object representing the system configuration.
:returns: a dictionary of the form {'<unique_attribute_value>': <smartstack replication hash>}
(the dictionary will contain keys for unique all attribute values)
"""
replication_info = {}
filtered_slaves = mesos_tools.get_all_slaves_for_blacklist_whitelist(
blacklist=blacklist,
whitelist=None,
)
if not filtered_slaves:
raise NoSlavesAvailableError
attribute_slave_dict = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=attribute,
)
full_name = compose_job_id(service, namespace)
for value, hosts in attribute_slave_dict.items():
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]['hostname']
repl_info = get_replication_for_services(
synapse_host=synapse_host,
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.
get_synapse_haproxy_url_format(),
services=[full_name],
)
replication_info[value] = repl_info
return replication_info
def test_get_mesos_slaves_grouped_by_attribute(mock_fetch_state):
fake_attribute = "fake_attribute"
fake_value_1 = "fake_value_1"
fake_value_2 = "fake_value_2"
mock_fetch_state.return_value = {
"slaves": [
{"hostname": "fake_host_1", "attributes": {"fake_attribute": fake_value_1}},
{"hostname": "fake_host_2", "attributes": {"fake_attribute": fake_value_2}},
{"hostname": "fake_host_3", "attributes": {"fake_attribute": fake_value_1}},
{"hostname": "fake_host_4", "attributes": {"fake_attribute": "fake_other_value"}},
]
}
expected = {
"fake_value_1": ["fake_host_1", "fake_host_3"],
"fake_value_2": ["fake_host_2"],
"fake_other_value": ["fake_host_4"],
}
actual = mesos_tools.get_mesos_slaves_grouped_by_attribute(fake_attribute)
assert actual == expected
def get_constraints(self, service_namespace_config):
"""Gets the constraints specified in the service's marathon configuration.
These are Marathon app constraints. See
https://mesosphere.github.io/marathon/docs/constraints.html
Defaults to `GROUP_BY region`. If the service's smartstack configuration
specifies a `discover` key, then defaults to `GROUP_BY <value of discover>` instead.
:param service_namespace_config: The service instance's configuration dictionary
:returns: The constraints specified in the config, or defaults described above
"""
if 'constraints' in self.config_dict:
return self.config_dict.get('constraints')
else:
discover_level = service_namespace_config.get_discover()
locations = get_mesos_slaves_grouped_by_attribute(
attribute=discover_level, blacklist=self.get_deploy_blacklist())
deploy_constraints = deploy_blacklist_to_constraints(self.get_deploy_blacklist())
routing_constraints = [[discover_level, "GROUP_BY", str(len(locations))]]
return routing_constraints + deploy_constraints
def get_routing_constraints(self, service_namespace_config):
"""
Returns a set of constraints in order to evenly group a marathon
application amongst instances of a discovery type.
If, for example, a given app's 'discover' key is set to 'region', then this function
computes the constraints required to group the app evenly amongst each
of the actual 'region' values in the cluster.
It does so by querying the value of the region attribute for each slave
in the cluster, returning a GROUP_BY constraint where the value is the
number of unique regions.
:param service_namespace_config: the config for this service
:returns: a set of constraints for marathon
"""
discover_level = service_namespace_config.get_discover()
slaves = get_slaves()
if not slaves:
raise NoSlavesAvailableError(
"No slaves could be found in the cluster."
)
filtered_slaves = filter_mesos_slaves_by_blacklist(
slaves=slaves,
blacklist=self.get_deploy_blacklist(),
whitelist=self.get_deploy_whitelist(),
)
if not filtered_slaves:
raise NoSlavesAvailableError(
("No suitable slaves could be found in the cluster for %s.%s"
"There are %d total slaves in the cluster, but after filtering"
" those available to the app according to the constraints set"
" by the deploy_blacklist and deploy_whitelist, there are 0"
" available.") % (self.service, self.instance, len(slaves))
)
value_dict = get_mesos_slaves_grouped_by_attribute(
filtered_slaves,
discover_level
)
routing_constraints = [[discover_level, "GROUP_BY", str(len(value_dict.keys()))]]
return routing_constraints
def status_smartstack_backends(service, instance, job_config, cluster, tasks, expected_count, soa_dir, verbose,
synapse_port, synapse_haproxy_url_format):
"""Returns detailed information about smartstack backends for a service
and instance.
return: A newline separated string of the smarststack backend status
"""
output = []
nerve_ns = marathon_tools.read_namespace_for_service_instance(service, instance, cluster)
service_instance = compose_job_id(service, nerve_ns)
if instance != nerve_ns:
ns_string = PaastaColors.bold(nerve_ns)
output.append("Smartstack: N/A - %s is announced in the %s namespace." % (instance, ns_string))
# If verbose mode is specified, then continue to show backends anyway, otherwise stop early
if not verbose:
return "\n".join(output)
service_namespace_config = marathon_tools.load_service_namespace_config(service, instance, soa_dir=soa_dir)
discover_location_type = service_namespace_config.get_discover()
monitoring_blacklist = job_config.get_monitoring_blacklist()
unique_attributes = get_mesos_slaves_grouped_by_attribute(
attribute=discover_location_type, blacklist=monitoring_blacklist)
if len(unique_attributes) == 0:
output.append("Smartstack: ERROR - %s is NOT in smartstack at all!" % service_instance)
else:
output.append("Smartstack:")
if verbose:
output.append(" Haproxy Service Name: %s" % service_instance)
output.append(" Backends:")
output.extend(pretty_print_smartstack_backends_for_locations(
service_instance,
tasks,
unique_attributes,
expected_count,
verbose,
synapse_port,
synapse_haproxy_url_format,
))
return "\n".join(output)
def get_routing_constraints(self, service_namespace_config):
"""
Returns a set of constraints in order to evenly group a marathon
application amongst instances of a discovery type.
If, for example, a given app's 'discover' key is set to 'region', then this function
computes the constraints required to group the app evenly amongst each
of the actual 'region' values in the cluster.
It does so by querying the value of the region attribute for each slave
in the cluster, returning a GROUP_BY constraint where the value is the
number of unique regions.
:param service_namespace_config: the config for this service
:returns: a set of constraints for marathon
"""
discover_level = service_namespace_config.get_discover()
slaves = get_slaves()
if not slaves:
raise NoSlavesAvailableError(
"No slaves could be found in the cluster."
)
filtered_slaves = filter_mesos_slaves_by_blacklist(
slaves=slaves,
blacklist=self.get_deploy_blacklist(),
whitelist=self.get_deploy_whitelist(),
)
if not filtered_slaves:
raise NoSlavesAvailableError(
("No suitable slaves could be found in the cluster for %s.%s"
"There are %d total slaves in the cluster, but after filtering"
" those available to the app according to the constraints set"
" by the deploy_blacklist and deploy_whitelist, there are 0"
" available.") % (self.service, self.instance, len(slaves))
)
value_dict = get_mesos_slaves_grouped_by_attribute(
filtered_slaves,
discover_level
)
routing_constraints = [[discover_level, "GROUP_BY", str(len(value_dict.keys()))]]
return routing_constraints
def get_smartstack_replication_for_attribute(attribute, service, namespace, blacklist, system_paasta_config):
"""Loads smartstack replication from a host with the specified attribute
:param attribute: a Mesos attribute
:param service: A service name, like 'example_service'
:param namespace: A particular smartstack namespace to inspect, like 'main'
:param constraints: A list of Marathon constraints to restrict which synapse hosts to query
:param blacklist: A list of blacklisted location tuples in the form of (location, value)
:param system_paasta_config: A SystemPaastaConfig object representing the system configuration.
:returns: a dictionary of the form {'<unique_attribute_value>': <smartstack replication hash>}
(the dictionary will contain keys for unique all attribute values)
"""
replication_info = {}
filtered_slaves = mesos_tools.get_all_slaves_for_blacklist_whitelist(
blacklist=blacklist,
whitelist=[],
)
if not filtered_slaves:
raise mesos_tools.NoSlavesAvailableError
attribute_slave_dict = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=attribute
)
full_name = compose_job_id(service, namespace)
for value, hosts in attribute_slave_dict.iteritems():
# arbitrarily choose the first host with a given attribute to query for replication stats
synapse_host = hosts[0]['hostname']
repl_info = get_replication_for_services(
synapse_host=synapse_host,
synapse_port=system_paasta_config.get_synapse_port(),
synapse_haproxy_url_format=system_paasta_config.get_synapse_haproxy_url_format(),
services=[full_name],
)
replication_info[value] = repl_info
return replication_info
def test_get_mesos_slaves_grouped_by_attribute(mock_fetch_state):
fake_attribute = 'fake_attribute'
fake_value_1 = 'fake_value_1'
fake_value_2 = 'fake_value_2'
mock_fetch_state.return_value = {
'slaves': [
{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': fake_value_1,
}
},
{
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': fake_value_2,
}
},
{
'hostname': 'fake_host_3',
'attributes': {
'fake_attribute': fake_value_1,
}
},
{
'hostname': 'fake_host_4',
'attributes': {
'fake_attribute': 'fake_other_value',
}
}
]
}
expected = {
'fake_value_1': ['fake_host_1', 'fake_host_3'],
'fake_value_2': ['fake_host_2'],
'fake_other_value': ['fake_host_4'],
}
actual = mesos_tools.get_mesos_slaves_grouped_by_attribute(fake_attribute)
assert actual == expected
def test_get_mesos_slaves_grouped_by_attribute_bombs_out_with_no_slaves(mock_fetch_state):
mock_fetch_state.return_value = {"slaves": []}
with raises(mesos_tools.NoSlavesAvailable):
mesos_tools.get_mesos_slaves_grouped_by_attribute("fake_attribute")
def test_get_mesos_slaves_grouped_by_attribute_bombs_out_with_no_slaves(mock_fetch_state):
mock_fetch_state.return_value = {
'slaves': []
}
with raises(mesos_tools.NoSlavesAvailable):
mesos_tools.get_mesos_slaves_grouped_by_attribute('fake_attribute')
def get_happy_tasks(app,
service,
nerve_ns,
system_paasta_config,
min_task_uptime=None,
check_haproxy=False):
"""Given a MarathonApp object, return the subset of tasks which are considered healthy.
With the default options, this returns tasks where at least one of the defined Marathon healthchecks passes.
For it to do anything interesting, set min_task_uptime or check_haproxy.
:param app: A MarathonApp object.
:param service: The name of the service.
:param nerve_ns: The nerve namespace
:param min_task_uptime: Minimum number of seconds that a task must be running before we consider it healthy. Useful
if tasks take a while to start up.
:param check_haproxy: Whether to check the local haproxy to make sure this task has been registered and discovered.
"""
tasks = app.tasks
happy = []
now = datetime.datetime.utcnow()
if check_haproxy:
tasks_in_smartstack = []
service_namespace = compose_job_id(service, nerve_ns)
service_namespace_config = marathon_tools.load_service_namespace_config(
service=service, namespace=nerve_ns)
discover_location_type = service_namespace_config.get_discover()
unique_values = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=mesos_tools.get_slaves(), attribute=discover_location_type)
for value, hosts in unique_values.iteritems():
synapse_hostname = hosts[0]['hostname']
tasks_in_smartstack.extend(
get_registered_marathon_tasks(
synapse_hostname,
system_paasta_config.get_synapse_port(),
system_paasta_config.get_synapse_haproxy_url_format(),
service_namespace,
tasks,
))
tasks = tasks_in_smartstack
for task in tasks:
if task.started_at is None:
# Can't be healthy if it hasn't started
continue
if min_task_uptime is not None:
if (now - task.started_at).total_seconds() < min_task_uptime:
continue
# if there are healthchecks defined for the app but none have executed yet, then task is unhappy
if len(app.health_checks) > 0 and len(task.health_check_results) == 0:
continue
# if there are health check results, check if at least one healthcheck is passing
if not marathon_tools.is_task_healthy(
task, require_all=False, default_healthy=True):
continue
happy.append(task)
return happy
def test_get_mesos_slaves_grouped_by_attribute():
fake_value_1 = 'fake_value_1'
fake_value_2 = 'fake_value_2'
fake_slaves = [
{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': fake_value_1,
}
},
{
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': fake_value_2,
}
},
{
'hostname': 'fake_host_3',
'attributes': {
'fake_attribute': fake_value_1,
}
},
{
'hostname': 'fake_host_4',
'attributes': {
'fake_attribute': 'fake_other_value',
}
}
]
expected = {
'fake_value_1': [
{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': fake_value_1,
}
},
{
'hostname': 'fake_host_3',
'attributes': {
'fake_attribute': fake_value_1,
}
},
],
'fake_value_2': [
{
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': fake_value_2,
}
},
],
'fake_other_value': [
{
'hostname': 'fake_host_4',
'attributes': {
'fake_attribute': 'fake_other_value',
}
}
]
}
actual = mesos_tools.get_mesos_slaves_grouped_by_attribute(fake_slaves, 'fake_attribute')
assert actual == expected
