def main(): args = parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARNING) system_paasta_config = load_system_paasta_config() cluster = system_paasta_config.get_cluster() clients = marathon_tools.get_marathon_clients( marathon_tools.get_marathon_servers(system_paasta_config)) all_clients = clients.get_all_clients() all_tasks = [] for client in all_clients: all_tasks.extend(client.list_tasks()) mesos_slaves = a_sync.block(get_slaves) smartstack_replication_checker = MesosSmartstackReplicationChecker( mesos_slaves, system_paasta_config) for service in list_services(soa_dir=args.soa_dir): service_config = PaastaServiceConfigLoader(service=service, soa_dir=args.soa_dir) for instance_config in service_config.instance_configs( cluster=cluster, instance_type_class=marathon_tools.MarathonServiceConfig): if instance_config.get_docker_image(): check_service_replication( instance_config=instance_config, all_tasks=all_tasks, smartstack_replication_checker= smartstack_replication_checker, ) else: log.debug( "%s is not deployed. Skipping replication monitoring." % instance_config.job_id)
def check_services_replication( soa_dir: str, cluster: str, service_instances: Sequence[str], instance_type_class: Type[InstanceConfig_T], check_service_replication: CheckServiceReplication, replication_checker: ReplicationChecker, all_tasks_or_pods: Sequence[Union[MarathonTask, V1Pod]], ) -> Tuple[int, int]: service_instances_set = set(service_instances) replication_statuses: List[bool] = [] for service in list_services(soa_dir=soa_dir): service_config = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir) for instance_config in service_config.instance_configs( cluster=cluster, instance_type_class=instance_type_class): if (service_instances_set and f"{service}{SPACER}{instance_config.instance}" not in service_instances_set): continue if instance_config.get_docker_image(): is_well_replicated = check_service_replication( instance_config=instance_config, all_tasks_or_pods=all_tasks_or_pods, replication_checker=replication_checker, ) if is_well_replicated is not None: replication_statuses.append(is_well_replicated) else: log.debug( "%s is not deployed. Skipping replication monitoring." % instance_config.job_id) num_under_replicated = len( [status for status in replication_statuses if status is False]) return num_under_replicated, len(replication_statuses)
def get_configs_of_services_to_scale( cluster: str, soa_dir: str = DEFAULT_SOA_DIR, services: Optional[Sequence[str]] = None, ) -> Sequence[MarathonServiceConfig]: if not services: services = list_services( soa_dir=soa_dir, ) configs = [] for service in services: service_config = PaastaServiceConfigLoader( service=service, soa_dir=soa_dir, ) for instance_config in service_config.instance_configs( cluster=cluster, instance_type_class=MarathonServiceConfig, ): if instance_config.get_max_instances() and instance_config.get_desired_state() == 'start' \ and instance_config.get_autoscaling_params()['decision_policy'] != 'bespoke': configs.append(instance_config) return configs
def check_all_kubernetes_based_services_replication( soa_dir: str, service_instances: Sequence[str], instance_type_class: Type[InstanceConfig_T], check_service_replication: CheckServiceReplication, namespace: str, ) -> None: kube_client = KubeClient() all_pods = get_all_pods(kube_client=kube_client, namespace=namespace) all_nodes = get_all_nodes(kube_client) system_paasta_config = load_system_paasta_config() cluster = system_paasta_config.get_cluster() smartstack_replication_checker = KubeSmartstackReplicationChecker( nodes=all_nodes, system_paasta_config=system_paasta_config) service_instances_set = set(service_instances) for service in list_services(soa_dir=soa_dir): service_config = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir) for instance_config in service_config.instance_configs( cluster=cluster, instance_type_class=instance_type_class): if (service_instances_set and f"{service}{SPACER}{instance_config.instance}" not in service_instances_set): continue if instance_config.get_docker_image(): check_service_replication( instance_config=instance_config, all_pods=all_pods, smartstack_replication_checker= smartstack_replication_checker, ) else: log.debug( "%s is not deployed. Skipping replication monitoring." % instance_config.job_id)
def completer_clusters(prefix, parsed_args, **kwargs): service = parsed_args.service or guess_service_name() if service in list_services(): return list_clusters(service) else: return list_clusters()
def apply_args_filters( args, ) -> Mapping[str, Mapping[str, Mapping[str, Type[InstanceConfig]]]]: """ Take an args object and returns the dict of cluster:service:instances Currently, will filter by clusters, instances, services, and deploy_groups If no instances are found, will print a message and try to find matching instances for each service :param args: args object containing attributes to filter by :returns: Dict of dicts, in format {cluster_name: {service_name: {instance1, instance2}}} """ clusters_services_instances: DefaultDict[ str, DefaultDict[str, Dict[str, Type[InstanceConfig]]] ] = defaultdict(lambda: defaultdict(dict)) if args.service: try: validate_service_name(args.service, soa_dir=args.soa_dir) except NoSuchService: paasta_print( PaastaColors.red(f'The service "{args.service}" does not exist.') ) all_services = list_services(soa_dir=args.soa_dir) suggestions = difflib.get_close_matches( args.service, all_services, n=5, cutoff=0.5 ) if suggestions: paasta_print(PaastaColors.red(f"Did you mean any of these?")) for suggestion in suggestions: paasta_print(PaastaColors.red(f" {suggestion}")) return clusters_services_instances all_services = [args.service] else: all_services = list_services(soa_dir=args.soa_dir) if args.service is None and args.owner is None: args.service = figure_out_service_name(args, soa_dir=args.soa_dir) if args.clusters: clusters = args.clusters.split(",") else: clusters = list_clusters() if args.instances: instances = args.instances.split(",") else: instances = None filters = get_filters(args) i_count = 0 for service in all_services: if args.service and service != args.service: continue for instance_conf in get_instance_configs_for_service( service, soa_dir=args.soa_dir, clusters=clusters, instances=instances ): if all([f(instance_conf) for f in filters]): cluster_service = clusters_services_instances[ instance_conf.get_cluster() ][service] cluster_service[instance_conf.get_instance()] = instance_conf.__class__ i_count += 1 if i_count == 0 and args.service and args.instances: for service in args.service.split(","): verify_instances(args.instances, service, clusters) return clusters_services_instances