def get_instance_config_for_service(soa_dir, service): for cluster in list_clusters( service=service, soa_dir=soa_dir, ): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, ) for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, )
def get_instance_configs_for_service(service, soa_dir, type_filter=None): for cluster in list_clusters( service=service, soa_dir=soa_dir, ): if type_filter is None: type_filter = ['marathon', 'chronos', 'adhoc'] if 'marathon' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir, ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'chronos' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', soa_dir=soa_dir, ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'adhoc' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='adhoc', soa_dir=soa_dir, ): yield load_adhoc_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, )
def get_services_to_k8s_namespaces( service_list: List[str], cluster: str, soa_dir: str, ) -> Dict[str, Set[str]]: services_to_k8s_namespaces: Dict[str, Set[str]] = defaultdict(set) for service in service_list: # Special handling for service `_shared`, since it doesn't actually exist # Copy shared secrest to all namespaces, assuming that if a secret is declared shared # the team is aware that more people can see it if service == "_shared": services_to_k8s_namespaces[service] = set( INSTANCE_TYPE_TO_K8S_NAMESPACE.values()) continue for instance_type in INSTANCE_TYPES: instances = get_service_instance_list( service=service, instance_type=instance_type, cluster=cluster, soa_dir=soa_dir, ) if instances: services_to_k8s_namespaces[service].add( INSTANCE_TYPE_TO_K8S_NAMESPACE[instance_type]) return dict(services_to_k8s_namespaces)
def test_get_service_instance_list(): fake_name = "hint" fake_instance_1 = "unsweet" fake_instance_2 = "water" fake_cluster = "16floz" fake_dir = "/nail/home/hipster" fake_job_config = {fake_instance_1: {}, fake_instance_2: {}} expected = [ (fake_name, fake_instance_1), (fake_name, fake_instance_1), (fake_name, fake_instance_2), (fake_name, fake_instance_2), ] with contextlib.nested( mock.patch( "paasta_tools.utils.service_configuration_lib.read_extra_service_information", autospec=True, return_value=fake_job_config, ) ) as (read_extra_info_patch,): actual = utils.get_service_instance_list(fake_name, fake_cluster, soa_dir=fake_dir) read_extra_info_patch.assert_any_call(fake_name, "marathon-16floz", soa_dir=fake_dir) read_extra_info_patch.assert_any_call(fake_name, "chronos-16floz", soa_dir=fake_dir) assert read_extra_info_patch.call_count == 2 assert sorted(expected) == sorted(actual)
def get_deploy_groups_used_by_framework(instance_type, service, soa_dir): """This is a kind of funny function that gets all the instances for specified service and framework, and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting. :param instance_type: one of 'marathon', 'chronos', 'adhoc' :param service: the service name :param soa_dir: The SOA configuration directory to read from :returns: a list of deploy group names used by the service. """ deploy_groups = [] for cluster in list_clusters(service, soa_dir): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type=instance_type, soa_dir=soa_dir, ): try: config = get_instance_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, instance_type=instance_type, ) deploy_groups.append(config.get_deploy_group()) except NotImplementedError: pass return deploy_groups
def validate_unique_instance_names(service_path): """Check that the service does not use the same instance name more than once""" soa_dir, service = path_to_soa_dir_service(service_path) check_passed = True for cluster in list_clusters(service, soa_dir): service_instances = get_service_instance_list(service=service, cluster=cluster, soa_dir=soa_dir) instance_names = [ service_instance[1] for service_instance in service_instances ] instance_name_to_count = Counter(instance_names) duplicate_instance_names = [ instance_name for instance_name, count in instance_name_to_count.items() if count > 1 ] if duplicate_instance_names: check_passed = False paasta_print( duplicate_instance_names_message( service, cluster, duplicate_instance_names, )) else: paasta_print(no_duplicate_instance_names_message(service, cluster)) return check_passed
def get_expected_instance_count_for_namespace(service, namespace, cluster=None, soa_dir=DEFAULT_SOA_DIR): """Get the number of expected instances for a namespace, based on the number of instances set to run on that namespace as specified in Marathon service configuration files. :param service: The service's name :param namespace: The namespace for that service to check :param soa_dir: The SOA configuration directory to read from :returns: An integer value of the # of expected instances for the namespace""" total_expected = 0 if not cluster: cluster = load_system_paasta_config().get_cluster() for name, instance in get_service_instance_list(service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir): srv_config = load_marathon_service_config(name, instance, cluster, soa_dir=soa_dir) instance_ns = srv_config.get_nerve_namespace() if namespace == instance_ns: total_expected += srv_config.get_instances() return total_expected
def test_get_service_instance_list(): fake_name = 'hint' fake_instance_1 = 'unsweet' fake_instance_2 = 'water' fake_cluster = '16floz' fake_dir = '/nail/home/hipster' fake_job_config = {fake_instance_1: {}, fake_instance_2: {}} expected = [(fake_name, fake_instance_1), (fake_name, fake_instance_1), (fake_name, fake_instance_2), (fake_name, fake_instance_2)] with contextlib.nested( mock.patch( 'paasta_tools.utils.service_configuration_lib.read_extra_service_information', autospec=True, return_value=fake_job_config), ) as (read_extra_info_patch, ): actual = utils.get_service_instance_list(fake_name, fake_cluster, soa_dir=fake_dir) read_extra_info_patch.assert_any_call(fake_name, 'marathon-16floz', soa_dir=fake_dir) read_extra_info_patch.assert_any_call(fake_name, 'chronos-16floz', soa_dir=fake_dir) assert read_extra_info_patch.call_count == 2 assert sorted(expected) == sorted(actual)
def list_job_names(service, cluster=None, soa_dir=DEFAULT_SOA_DIR): """A chronos-specific wrapper around utils.get_service_instance_list. :param name: The service name :param cluster: The cluster to read the configuration for :param soa_dir: The SOA config directory to read from :returns: A list of tuples of (name, job) for each job defined for the service name """ return get_service_instance_list(service, cluster, 'chronos', soa_dir)
def get_instance_configs_for_service(service, soa_dir): for cluster in list_clusters( service=service, soa_dir=soa_dir, ): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir, ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', soa_dir=soa_dir, ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='adhoc', soa_dir=soa_dir, ): yield load_adhoc_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, )
def get_marathon_steps(service): """This is a kind of funny function that gets all the marathon instances for a service and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting.""" steps = [] for cluster in list_clusters(service): for instance in get_service_instance_list(service, cluster=cluster, instance_type='marathon'): steps.append("%s.%s" % (cluster, instance[1])) return steps
def get_marathon_steps(service, soa_dir): """This is a kind of funny function that gets all the marathon instances for a service and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting.""" steps = [] for cluster in list_clusters(service, soa_dir): for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type="marathon", soa_dir=soa_dir ): config = load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False ) steps.append(config.get_deploy_group()) return steps
def get_expected_instance_count_for_namespace(service, namespace, cluster=None, soa_dir=DEFAULT_SOA_DIR): """Get the number of expected instances for a namespace, based on the number of instances set to run on that namespace as specified in Marathon service configuration files. :param service: The service's name :param namespace: The namespace for that service to check :param soa_dir: The SOA configuration directory to read from :returns: An integer value of the # of expected instances for the namespace""" total_expected = 0 if not cluster: cluster = load_system_paasta_config().get_cluster() for name, instance in get_service_instance_list( service, cluster=cluster, instance_type="marathon", soa_dir=soa_dir ): srv_config = load_marathon_service_config(name, instance, cluster, soa_dir=soa_dir) instance_ns = srv_config.get_nerve_namespace() if namespace == instance_ns: total_expected += srv_config.get_instances() return total_expected
def get_marathon_steps(service, soa_dir): """This is a kind of funny function that gets all the marathon instances for a service and massages it into a form that matches up with what deploy.yaml's steps look like. This is only so we can compare it 1-1 with what deploy.yaml has for linting.""" steps = [] for cluster in list_clusters(service, soa_dir): for _, instance in get_service_instance_list(service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir): config = load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) steps.append(config.get_deploy_group()) return steps
def test_get_service_instance_list(): fake_name = 'hint' fake_instance_1 = 'unsweet' fake_instance_2 = 'water' fake_cluster = '16floz' fake_dir = '/nail/home/hipster' fake_job_config = {fake_instance_1: {}, fake_instance_2: {}} expected = [(fake_name, fake_instance_1), (fake_name, fake_instance_1), (fake_name, fake_instance_2), (fake_name, fake_instance_2)] with contextlib.nested( mock.patch('paasta_tools.utils.service_configuration_lib.read_extra_service_information', autospec=True, return_value=fake_job_config), ) as ( read_extra_info_patch, ): actual = utils.get_service_instance_list(fake_name, fake_cluster, soa_dir=fake_dir) read_extra_info_patch.assert_any_call(fake_name, 'marathon-16floz', soa_dir=fake_dir) read_extra_info_patch.assert_any_call(fake_name, 'chronos-16floz', soa_dir=fake_dir) assert read_extra_info_patch.call_count == 2 assert sorted(expected) == sorted(actual)
def get_instance_configs_for_service( service: str, soa_dir: str, type_filter: Optional[Sequence[str]] = None, ) -> Iterable[InstanceConfig]: for cluster in list_clusters( service=service, soa_dir=soa_dir, ): if type_filter is None: type_filter = ['marathon', 'chronos', 'adhoc', 'kubernetes'] if 'marathon' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='marathon', soa_dir=soa_dir, ): yield load_marathon_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'chronos' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='chronos', soa_dir=soa_dir, ): yield load_chronos_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'adhoc' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='adhoc', soa_dir=soa_dir, ): yield load_adhoc_job_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, ) if 'kubernetes' in type_filter: for _, instance in get_service_instance_list( service=service, cluster=cluster, instance_type='kubernetes', soa_dir=soa_dir, ): yield load_kubernetes_service_config( service=service, instance=instance, cluster=cluster, soa_dir=soa_dir, load_deployments=False, )