def _get_allowed_locations_and_hostnames(
self, instance_config) -> Dict[str, list]:
"""Returns a dict of locations and lists of corresponding mesos slaves
where deployment of the instance is allowed.
:param instance_config: An instance of MarathonServiceConfig
:returns: A dict {"uswest1-prod": ['hostname1', 'hostname2], ...}.
"""
monitoring_blacklist = instance_config.get_monitoring_blacklist(
system_deploy_blacklist=self._system_paasta_config.
get_deploy_blacklist(), )
filtered_slaves = mesos_tools.filter_mesos_slaves_by_blacklist(
slaves=self._mesos_slaves,
blacklist=monitoring_blacklist,
whitelist=None,
)
discover_location_type = marathon_tools.load_service_namespace_config(
service=instance_config.service,
namespace=instance_config.instance,
soa_dir=instance_config.soa_dir,
).get_discover()
slaves_grouped_by_attribute = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
return {
attr: [s['hostname'] for s in slaves]
for attr, slaves in slaves_grouped_by_attribute.items()
}
def _get_allowed_locations_and_hosts(
self, instance_config: InstanceConfig
) -> Dict[str, Sequence[SmartstackHost]]:
"""Returns a dict of locations and lists of corresponding mesos slaves
where deployment of the instance is allowed.
:param instance_config: An instance of MarathonServiceConfig
:returns: A dict {"uswest1-prod": [SmartstackHost(), SmartstackHost(), ...]}
"""
monitoring_blacklist = instance_config.get_monitoring_blacklist(
system_deploy_blacklist=self._system_paasta_config.
get_deploy_blacklist(), )
filtered_slaves = mesos_tools.filter_mesos_slaves_by_blacklist(
slaves=self._mesos_slaves,
blacklist=monitoring_blacklist,
whitelist=None,
)
discover_location_type = marathon_tools.load_service_namespace_config(
service=instance_config.service,
namespace=instance_config.instance,
soa_dir=instance_config.soa_dir,
).get_discover()
attribute_to_slaves = mesos_tools.get_mesos_slaves_grouped_by_attribute(
slaves=filtered_slaves,
attribute=discover_location_type,
)
ret: Dict[str, Sequence[SmartstackHost]] = {}
for attr, slaves in attribute_to_slaves.items():
ret[attr] = [
SmartstackHost(hostname=slave['hostname'],
pool=slave['attributes']['pool'])
for slave in slaves
]
return ret
def test_filter_mesos_slaves_by_blacklist_when_filtered(mock_slave_passes_blacklist):
mock_slave_passes_blacklist.return_value = False
slaves = [
{"hostname": "fake_host_1", "attributes": {"fake_attribute": "fake_value_1"}},
{"hostname": "fake_host_2", "attributes": {"fake_attribute": "fake_value_1"}},
]
blacklist = []
actual = mesos_tools.filter_mesos_slaves_by_blacklist(slaves=slaves, blacklist=blacklist)
assert mock_slave_passes_blacklist.call_count == 2
assert actual == []
def test_filter_mesos_slaves_by_blacklist_when_filtered(mock_slave_passes_blacklist):
mock_slave_passes_blacklist.return_value = False
slaves = [
{"hostname": "fake_host_1", "attributes": {"fake_attribute": "fake_value_1"}},
{"hostname": "fake_host_2", "attributes": {"fake_attribute": "fake_value_1"}},
]
blacklist = []
whitelist = None
actual = mesos_tools.filter_mesos_slaves_by_blacklist(
slaves=slaves, blacklist=blacklist, whitelist=whitelist
)
assert mock_slave_passes_blacklist.call_count == 2
assert actual == []
def get_routing_constraints(self, service_namespace_config,
system_paasta_config):
"""
Returns a set of constraints in order to evenly group a marathon
application amongst instances of a discovery type.
If, for example, a given app's 'discover' key is set to 'region', then this function
computes the constraints required to group the app evenly amongst each
of the actual 'region' values in the cluster.
It does so by querying the value of the discover attribute for each expected slave in the cluster (as defined
by the expected_slave_attributes key in system paasta config), returning a GROUP_BY constraint where the value
is the number of unique values for that attribute.
If you have not set expected_slave_attributes in the system paasta config, this function returns an empty list.
:param service_namespace_config: the config for this service
:returns: a set of constraints for marathon
"""
discover_level = service_namespace_config.get_discover()
expected_slave_attributes = system_paasta_config.get_expected_slave_attributes(
)
if expected_slave_attributes is None:
return []
fake_slaves = [{"attributes": a} for a in expected_slave_attributes]
filtered_slaves = filter_mesos_slaves_by_blacklist(
slaves=fake_slaves,
blacklist=self.get_deploy_blacklist(
system_deploy_blacklist=system_paasta_config.
get_deploy_blacklist(), ),
whitelist=self.get_deploy_whitelist(
system_deploy_whitelist=system_paasta_config.
get_deploy_whitelist(), ),
)
if not filtered_slaves:
raise NoSlavesAvailableError((
"We do not believe any slaves on the cluster will match the constraints for %s.%s. If you believe "
"this is incorrect, have your system administrator adjust the value of expected_slave_attributes "
"in the system paasta configs.") %
(self.service, self.instance), )
value_dict = get_mesos_slaves_grouped_by_attribute(
filtered_slaves,
discover_level,
)
routing_constraints = [[
discover_level, "GROUP_BY",
str(len(value_dict.keys()))
]]
return routing_constraints
def test_filter_mesos_slaves_by_blacklist_when_filtered(
mock_slave_passes_blacklist):
mock_slave_passes_blacklist.return_value = False
slaves = [{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': 'fake_value_1',
}
}, {
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': 'fake_value_1',
}
}]
blacklist = []
actual = mesos_tools.filter_mesos_slaves_by_blacklist(slaves=slaves,
blacklist=blacklist)
assert mock_slave_passes_blacklist.call_count == 2
assert actual == []
def test_filter_mesos_slaves_by_blacklist_when_filtered(mock_slave_passes_blacklist):
mock_slave_passes_blacklist.return_value = False
slaves = [
{
'hostname': 'fake_host_1',
'attributes': {
'fake_attribute': 'fake_value_1',
}
},
{
'hostname': 'fake_host_2',
'attributes': {
'fake_attribute': 'fake_value_1',
}
}
]
blacklist = []
actual = mesos_tools.filter_mesos_slaves_by_blacklist(slaves=slaves, blacklist=blacklist)
assert mock_slave_passes_blacklist.call_count == 2
assert actual == []
def get_routing_constraints(self, service_namespace_config):
"""
Returns a set of constraints in order to evenly group a marathon
application amongst instances of a discovery type.
If, for example, a given app's 'discover' key is set to 'region', then this function
computes the constraints required to group the app evenly amongst each
of the actual 'region' values in the cluster.
It does so by querying the value of the region attribute for each slave
in the cluster, returning a GROUP_BY constraint where the value is the
number of unique regions.
:param service_namespace_config: the config for this service
:returns: a set of constraints for marathon
"""
discover_level = service_namespace_config.get_discover()
slaves = get_slaves()
if not slaves:
raise NoSlavesAvailableError(
"No slaves could be found in the cluster."
)
filtered_slaves = filter_mesos_slaves_by_blacklist(
slaves=slaves,
blacklist=self.get_deploy_blacklist(),
whitelist=self.get_deploy_whitelist(),
)
if not filtered_slaves:
raise NoSlavesAvailableError(
("No suitable slaves could be found in the cluster for %s.%s"
"There are %d total slaves in the cluster, but after filtering"
" those available to the app according to the constraints set"
" by the deploy_blacklist and deploy_whitelist, there are 0"
" available.") % (self.service, self.instance, len(slaves))
)
value_dict = get_mesos_slaves_grouped_by_attribute(
filtered_slaves,
discover_level
)
routing_constraints = [[discover_level, "GROUP_BY", str(len(value_dict.keys()))]]
return routing_constraints
def get_routing_constraints(self, service_namespace_config):
"""
Returns a set of constraints in order to evenly group a marathon
application amongst instances of a discovery type.
If, for example, a given app's 'discover' key is set to 'region', then this function
computes the constraints required to group the app evenly amongst each
of the actual 'region' values in the cluster.
It does so by querying the value of the region attribute for each slave
in the cluster, returning a GROUP_BY constraint where the value is the
number of unique regions.
:param service_namespace_config: the config for this service
:returns: a set of constraints for marathon
"""
discover_level = service_namespace_config.get_discover()
slaves = get_slaves()
if not slaves:
raise NoSlavesAvailableError(
"No slaves could be found in the cluster."
)
filtered_slaves = filter_mesos_slaves_by_blacklist(
slaves=slaves,
blacklist=self.get_deploy_blacklist(),
whitelist=self.get_deploy_whitelist(),
)
if not filtered_slaves:
raise NoSlavesAvailableError(
("No suitable slaves could be found in the cluster for %s.%s"
"There are %d total slaves in the cluster, but after filtering"
" those available to the app according to the constraints set"
" by the deploy_blacklist and deploy_whitelist, there are 0"
" available.") % (self.service, self.instance, len(slaves))
)
value_dict = get_mesos_slaves_grouped_by_attribute(
filtered_slaves,
discover_level
)
routing_constraints = [[discover_level, "GROUP_BY", str(len(value_dict.keys()))]]
return routing_constraints
