def get_instance_config_for_service(soa_dir, service):
for cluster in list_clusters(
service=service,
soa_dir=soa_dir,
):
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='marathon',
):
yield load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='chronos',
):
yield load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
def get_instance_config_for_service(soa_dir, service):
for cluster in list_clusters(
service=service,
soa_dir=soa_dir,
):
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='marathon',
):
yield load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='chronos',
):
yield load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
)
def get_instance_configs_for_service(service, soa_dir, type_filter=None):
for cluster in list_clusters(
service=service,
soa_dir=soa_dir,
):
if type_filter is None:
type_filter = ['marathon', 'chronos', 'adhoc']
if 'marathon' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='marathon',
soa_dir=soa_dir,
):
yield load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if 'chronos' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='chronos',
soa_dir=soa_dir,
):
yield load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if 'adhoc' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='adhoc',
soa_dir=soa_dir,
):
yield load_adhoc_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
def get_services_to_k8s_namespaces(
service_list: List[str],
cluster: str,
soa_dir: str,
) -> Dict[str, Set[str]]:
services_to_k8s_namespaces: Dict[str, Set[str]] = defaultdict(set)
for service in service_list:
# Special handling for service `_shared`, since it doesn't actually exist
# Copy shared secrest to all namespaces, assuming that if a secret is declared shared
# the team is aware that more people can see it
if service == "_shared":
services_to_k8s_namespaces[service] = set(
INSTANCE_TYPE_TO_K8S_NAMESPACE.values())
continue
for instance_type in INSTANCE_TYPES:
instances = get_service_instance_list(
service=service,
instance_type=instance_type,
cluster=cluster,
soa_dir=soa_dir,
)
if instances:
services_to_k8s_namespaces[service].add(
INSTANCE_TYPE_TO_K8S_NAMESPACE[instance_type])
return dict(services_to_k8s_namespaces)
def test_get_service_instance_list():
fake_name = "hint"
fake_instance_1 = "unsweet"
fake_instance_2 = "water"
fake_cluster = "16floz"
fake_dir = "/nail/home/hipster"
fake_job_config = {fake_instance_1: {}, fake_instance_2: {}}
expected = [
(fake_name, fake_instance_1),
(fake_name, fake_instance_1),
(fake_name, fake_instance_2),
(fake_name, fake_instance_2),
]
with contextlib.nested(
mock.patch(
"paasta_tools.utils.service_configuration_lib.read_extra_service_information",
autospec=True,
return_value=fake_job_config,
)
) as (read_extra_info_patch,):
actual = utils.get_service_instance_list(fake_name, fake_cluster, soa_dir=fake_dir)
read_extra_info_patch.assert_any_call(fake_name, "marathon-16floz", soa_dir=fake_dir)
read_extra_info_patch.assert_any_call(fake_name, "chronos-16floz", soa_dir=fake_dir)
assert read_extra_info_patch.call_count == 2
assert sorted(expected) == sorted(actual)
def get_deploy_groups_used_by_framework(instance_type, service, soa_dir):
"""This is a kind of funny function that gets all the instances for specified
service and framework, and massages it into a form that matches up with what
deploy.yaml's steps look like. This is only so we can compare it 1-1
with what deploy.yaml has for linting.
:param instance_type: one of 'marathon', 'chronos', 'adhoc'
:param service: the service name
:param soa_dir: The SOA configuration directory to read from
:returns: a list of deploy group names used by the service.
"""
deploy_groups = []
for cluster in list_clusters(service, soa_dir):
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type=instance_type,
soa_dir=soa_dir,
):
try:
config = get_instance_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
instance_type=instance_type,
)
deploy_groups.append(config.get_deploy_group())
except NotImplementedError:
pass
return deploy_groups
def validate_unique_instance_names(service_path):
"""Check that the service does not use the same instance name more than once"""
soa_dir, service = path_to_soa_dir_service(service_path)
check_passed = True
for cluster in list_clusters(service, soa_dir):
service_instances = get_service_instance_list(service=service,
cluster=cluster,
soa_dir=soa_dir)
instance_names = [
service_instance[1] for service_instance in service_instances
]
instance_name_to_count = Counter(instance_names)
duplicate_instance_names = [
instance_name
for instance_name, count in instance_name_to_count.items()
if count > 1
]
if duplicate_instance_names:
check_passed = False
paasta_print(
duplicate_instance_names_message(
service,
cluster,
duplicate_instance_names,
))
else:
paasta_print(no_duplicate_instance_names_message(service, cluster))
return check_passed
def get_expected_instance_count_for_namespace(service,
namespace,
cluster=None,
soa_dir=DEFAULT_SOA_DIR):
"""Get the number of expected instances for a namespace, based on the number
of instances set to run on that namespace as specified in Marathon service
configuration files.
:param service: The service's name
:param namespace: The namespace for that service to check
:param soa_dir: The SOA configuration directory to read from
:returns: An integer value of the # of expected instances for the namespace"""
total_expected = 0
if not cluster:
cluster = load_system_paasta_config().get_cluster()
for name, instance in get_service_instance_list(service,
cluster=cluster,
instance_type='marathon',
soa_dir=soa_dir):
srv_config = load_marathon_service_config(name,
instance,
cluster,
soa_dir=soa_dir)
instance_ns = srv_config.get_nerve_namespace()
if namespace == instance_ns:
total_expected += srv_config.get_instances()
return total_expected
def test_get_service_instance_list():
fake_name = 'hint'
fake_instance_1 = 'unsweet'
fake_instance_2 = 'water'
fake_cluster = '16floz'
fake_dir = '/nail/home/hipster'
fake_job_config = {fake_instance_1: {}, fake_instance_2: {}}
expected = [(fake_name, fake_instance_1), (fake_name, fake_instance_1),
(fake_name, fake_instance_2), (fake_name, fake_instance_2)]
with contextlib.nested(
mock.patch(
'paasta_tools.utils.service_configuration_lib.read_extra_service_information',
autospec=True,
return_value=fake_job_config), ) as (read_extra_info_patch, ):
actual = utils.get_service_instance_list(fake_name,
fake_cluster,
soa_dir=fake_dir)
read_extra_info_patch.assert_any_call(fake_name,
'marathon-16floz',
soa_dir=fake_dir)
read_extra_info_patch.assert_any_call(fake_name,
'chronos-16floz',
soa_dir=fake_dir)
assert read_extra_info_patch.call_count == 2
assert sorted(expected) == sorted(actual)
def list_job_names(service, cluster=None, soa_dir=DEFAULT_SOA_DIR):
"""A chronos-specific wrapper around utils.get_service_instance_list.
:param name: The service name
:param cluster: The cluster to read the configuration for
:param soa_dir: The SOA config directory to read from
:returns: A list of tuples of (name, job) for each job defined for the service name
"""
return get_service_instance_list(service, cluster, 'chronos', soa_dir)
def list_job_names(service, cluster=None, soa_dir=DEFAULT_SOA_DIR):
"""A chronos-specific wrapper around utils.get_service_instance_list.
:param name: The service name
:param cluster: The cluster to read the configuration for
:param soa_dir: The SOA config directory to read from
:returns: A list of tuples of (name, job) for each job defined for the service name
"""
return get_service_instance_list(service, cluster, 'chronos', soa_dir)
def get_instance_configs_for_service(service, soa_dir):
for cluster in list_clusters(
service=service,
soa_dir=soa_dir,
):
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='marathon',
soa_dir=soa_dir,
):
yield load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='chronos',
soa_dir=soa_dir,
):
yield load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='adhoc',
soa_dir=soa_dir,
):
yield load_adhoc_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
def get_instance_configs_for_service(service, soa_dir):
for cluster in list_clusters(
service=service,
soa_dir=soa_dir,
):
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='marathon',
soa_dir=soa_dir,
):
yield load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='chronos',
soa_dir=soa_dir,
):
yield load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='adhoc',
soa_dir=soa_dir,
):
yield load_adhoc_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
def get_marathon_steps(service):
"""This is a kind of funny function that gets all the marathon instances
for a service and massages it into a form that matches up with what
deploy.yaml's steps look like. This is only so we can compare it 1-1
with what deploy.yaml has for linting."""
steps = []
for cluster in list_clusters(service):
for instance in get_service_instance_list(service, cluster=cluster, instance_type='marathon'):
steps.append("%s.%s" % (cluster, instance[1]))
return steps
def get_marathon_steps(service):
"""This is a kind of funny function that gets all the marathon instances
for a service and massages it into a form that matches up with what
deploy.yaml's steps look like. This is only so we can compare it 1-1
with what deploy.yaml has for linting."""
steps = []
for cluster in list_clusters(service):
for instance in get_service_instance_list(service, cluster=cluster, instance_type='marathon'):
steps.append("%s.%s" % (cluster, instance[1]))
return steps
def get_marathon_steps(service, soa_dir):
"""This is a kind of funny function that gets all the marathon instances
for a service and massages it into a form that matches up with what
deploy.yaml's steps look like. This is only so we can compare it 1-1
with what deploy.yaml has for linting."""
steps = []
for cluster in list_clusters(service, soa_dir):
for _, instance in get_service_instance_list(
service=service, cluster=cluster, instance_type="marathon", soa_dir=soa_dir
):
config = load_marathon_service_config(
service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False
)
steps.append(config.get_deploy_group())
return steps
def get_expected_instance_count_for_namespace(service, namespace, cluster=None, soa_dir=DEFAULT_SOA_DIR):
"""Get the number of expected instances for a namespace, based on the number
of instances set to run on that namespace as specified in Marathon service
configuration files.
:param service: The service's name
:param namespace: The namespace for that service to check
:param soa_dir: The SOA configuration directory to read from
:returns: An integer value of the # of expected instances for the namespace"""
total_expected = 0
if not cluster:
cluster = load_system_paasta_config().get_cluster()
for name, instance in get_service_instance_list(
service, cluster=cluster, instance_type="marathon", soa_dir=soa_dir
):
srv_config = load_marathon_service_config(name, instance, cluster, soa_dir=soa_dir)
instance_ns = srv_config.get_nerve_namespace()
if namespace == instance_ns:
total_expected += srv_config.get_instances()
return total_expected
def get_marathon_steps(service, soa_dir):
"""This is a kind of funny function that gets all the marathon instances
for a service and massages it into a form that matches up with what
deploy.yaml's steps look like. This is only so we can compare it 1-1
with what deploy.yaml has for linting."""
steps = []
for cluster in list_clusters(service, soa_dir):
for _, instance in get_service_instance_list(service=service,
cluster=cluster,
instance_type='marathon',
soa_dir=soa_dir):
config = load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
steps.append(config.get_deploy_group())
return steps
def test_get_service_instance_list():
fake_name = 'hint'
fake_instance_1 = 'unsweet'
fake_instance_2 = 'water'
fake_cluster = '16floz'
fake_dir = '/nail/home/hipster'
fake_job_config = {fake_instance_1: {},
fake_instance_2: {}}
expected = [(fake_name, fake_instance_1), (fake_name, fake_instance_1),
(fake_name, fake_instance_2), (fake_name, fake_instance_2)]
with contextlib.nested(
mock.patch('paasta_tools.utils.service_configuration_lib.read_extra_service_information', autospec=True,
return_value=fake_job_config),
) as (
read_extra_info_patch,
):
actual = utils.get_service_instance_list(fake_name, fake_cluster, soa_dir=fake_dir)
read_extra_info_patch.assert_any_call(fake_name, 'marathon-16floz', soa_dir=fake_dir)
read_extra_info_patch.assert_any_call(fake_name, 'chronos-16floz', soa_dir=fake_dir)
assert read_extra_info_patch.call_count == 2
assert sorted(expected) == sorted(actual)
def get_instance_configs_for_service(
service: str,
soa_dir: str,
type_filter: Optional[Sequence[str]] = None,
) -> Iterable[InstanceConfig]:
for cluster in list_clusters(
service=service,
soa_dir=soa_dir,
):
if type_filter is None:
type_filter = ['marathon', 'chronos', 'adhoc', 'kubernetes']
if 'marathon' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='marathon',
soa_dir=soa_dir,
):
yield load_marathon_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if 'chronos' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='chronos',
soa_dir=soa_dir,
):
yield load_chronos_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if 'adhoc' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='adhoc',
soa_dir=soa_dir,
):
yield load_adhoc_job_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
if 'kubernetes' in type_filter:
for _, instance in get_service_instance_list(
service=service,
cluster=cluster,
instance_type='kubernetes',
soa_dir=soa_dir,
):
yield load_kubernetes_service_config(
service=service,
instance=instance,
cluster=cluster,
soa_dir=soa_dir,
load_deployments=False,
)
