def pretty_print_smartstack_backends_for_locations(service_instance, tasks,
                                                   locations, expected_count,
                                                   verbose):
    """
    Pretty prints the status of smartstack backends of a specified service and instance in the specified locations
    """
    rows = [("      Name", "LastCheck", "LastChange", "Status")]
    expected_count_per_location = int(expected_count / len(locations))
    for location in sorted(locations):
        hosts = locations[location]
        # arbitrarily choose the first host with a given attribute to query for replication stats
        synapse_host = hosts[0]
        sorted_backends = sorted(
            get_backends(service_instance,
                         synapse_host=synapse_host,
                         synapse_port=DEFAULT_SYNAPSE_PORT),
            key=lambda backend: backend['status'],
            reverse=True
        )  # Specify reverse so that backends in 'UP' are placed above 'MAINT'
        matched_tasks = match_backends_and_tasks(sorted_backends, tasks)
        running_count = sum(1 for backend, task in matched_tasks
                            if backend and backend_is_up(backend))
        rows.append(
            "    %s - %s" % (location,
                             haproxy_backend_report(
                                 expected_count_per_location, running_count)))

        # If verbose mode is specified, show status of individual backends
        if verbose:
            for backend, task in matched_tasks:
                if backend is not None:
                    rows.append(
                        format_haproxy_backend_row(backend, task is not None))
    return format_table(rows)
def pretty_print_smartstack_backends_for_locations(service_instance, tasks, locations, expected_count, verbose,
                                                   synapse_port, synapse_haproxy_url_format):
    """
    Pretty prints the status of smartstack backends of a specified service and instance in the specified locations
    """
    rows = [("      Name", "LastCheck", "LastChange", "Status")] if verbose else []
    expected_count_per_location = int(expected_count / len(locations))
    for location in sorted(locations):
        hosts = locations[location]
        # arbitrarily choose the first host with a given attribute to query for replication stats
        synapse_host = hosts[0]
        sorted_backends = sorted(
            get_backends(
                service_instance,
                synapse_host=synapse_host,
                synapse_port=synapse_port,
                synapse_haproxy_url_format=synapse_haproxy_url_format,
            ),
            key=lambda backend: backend['status'],
            reverse=True,  # Specify reverse so that backends in 'UP' are placed above 'MAINT'
        )
        matched_tasks = match_backends_and_tasks(sorted_backends, tasks)
        running_count = sum(1 for backend, task in matched_tasks if backend and backend_is_up(backend))
        rows.append("    %s - %s" % (location, haproxy_backend_report(expected_count_per_location, running_count)))

        # If verbose mode is specified, show status of individual backends
        if verbose:
            for backend, task in matched_tasks:
                if backend is not None:
                    rows.append(format_haproxy_backend_row(backend, task is not None))
    return format_table(rows)
Exemple #3
0
def are_local_tasks_in_danger():
    try:
        system_paasta_config = utils.load_system_paasta_config()
        local_services = marathon_services_running_here()
        local_backends = get_backends(
            service=None,
            synapse_host=system_paasta_config.get_default_synapse_host(),
            synapse_port=system_paasta_config.get_synapse_port(),
            synapse_haproxy_url_format=system_paasta_config.
            get_synapse_haproxy_url_format(),
        )
        for service, instance, port in local_services:
            log.info(f"Inspecting {service}.{instance} on {port}")
            if is_healthy_in_haproxy(port, local_backends) and \
               synapse_replication_is_low(service, instance, system_paasta_config, local_backends=local_backends):
                log.warning(
                    "{}.{} on port {} is healthy but the service is in danger!"
                    .format(
                        service,
                        instance,
                        port,
                    ))
                return True
        return False
    except Exception:
        log.warning(traceback.format_exc())
        return False
Exemple #4
0
def _build_smartstack_location_dict(
    synapse_host: str,
    synapse_port: int,
    synapse_haproxy_url_format: str,
    registration: str,
    pods: Iterable[V1Pod],
    location: str,
    should_return_individual_backends: bool,
) -> MutableMapping[str, Any]:
    sorted_backends = sorted(
        smartstack_tools.get_backends(
            registration,
            synapse_host=synapse_host,
            synapse_port=synapse_port,
            synapse_haproxy_url_format=synapse_haproxy_url_format,
        ),
        key=lambda backend: backend["status"],
        reverse=True,  # put 'UP' backends above 'MAINT' backends
    )

    matched_backends_and_pods = match_backends_and_pods(sorted_backends, pods)
    location_dict = smartstack_tools.build_smartstack_location_dict(
        location, matched_backends_and_pods, should_return_individual_backends
    )
    return location_dict
Exemple #5
0
def _build_smartstack_location_dict_for_backends(
    synapse_host: str,
    registration: str,
    tasks: Sequence[MarathonTask],
    location: str,
    should_return_individual_backends: bool,
) -> MutableMapping[str, Any]:
    sorted_smartstack_backends = sorted(
        smartstack_tools.get_backends(
            registration,
            synapse_host=synapse_host,
            synapse_port=settings.system_paasta_config.get_synapse_port(),
            synapse_haproxy_url_format=settings.system_paasta_config.
            get_synapse_haproxy_url_format(),
        ),
        key=lambda backend: backend["status"],
        reverse=True,  # put 'UP' backends above 'MAINT' backends
    )
    matched_smartstack_backends_and_tasks = smartstack_tools.match_backends_and_tasks(
        sorted_smartstack_backends, tasks)
    return smartstack_tools.build_smartstack_location_dict(
        location,
        matched_smartstack_backends_and_tasks,
        should_return_individual_backends,
    )
Exemple #6
0
def smartstack_status(
    service: str,
    instance: str,
    job_config: LongRunningServiceConfig,
    service_namespace_config: ServiceNamespaceConfig,
    pods: Sequence[V1Pod],
    settings: Any,
    should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:

    registration = job_config.get_registrations()[0]
    instance_pool = job_config.get_pool()

    smartstack_replication_checker = KubeSmartstackReplicationChecker(
        nodes=kubernetes_tools.get_all_nodes(settings.kubernetes_client),
        system_paasta_config=settings.system_paasta_config,
    )
    node_hostname_by_location = smartstack_replication_checker.get_allowed_locations_and_hosts(
        job_config
    )

    expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
        service=service,
        namespace=instance,
        cluster=settings.cluster,
        instance_type_class=KubernetesDeploymentConfig,
    )
    expected_count_per_location = int(
        expected_smartstack_count / len(node_hostname_by_location)
    )
    smartstack_status: MutableMapping[str, Any] = {
        "registration": registration,
        "expected_backends_per_location": expected_count_per_location,
        "locations": [],
    }

    for location, hosts in node_hostname_by_location.items():
        synapse_host = smartstack_replication_checker.get_first_host_in_pool(
            hosts, instance_pool
        )
        sorted_backends = sorted(
            smartstack_tools.get_backends(
                registration,
                synapse_host=synapse_host,
                synapse_port=settings.system_paasta_config.get_synapse_port(),
                synapse_haproxy_url_format=settings.system_paasta_config.get_synapse_haproxy_url_format(),
            ),
            key=lambda backend: backend["status"],
            reverse=True,  # put 'UP' backends above 'MAINT' backends
        )

        matched_backends_and_pods = match_backends_and_pods(sorted_backends, pods)
        location_dict = smartstack_tools.build_smartstack_location_dict(
            location, matched_backends_and_pods, should_return_individual_backends
        )
        smartstack_status["locations"].append(location_dict)

    return smartstack_status
Exemple #7
0
def marathon_smartstack_status(
    service: str,
    instance: str,
    job_config: marathon_tools.MarathonServiceConfig,
    service_namespace_config: ServiceNamespaceConfig,
    tasks: Sequence[MarathonTask],
    should_return_individual_backends: bool = False,
) -> Mapping[str, Any]:
    registration = job_config.get_registrations()[0]
    discover_location_type = service_namespace_config.get_discover()
    monitoring_blacklist = job_config.get_monitoring_blacklist(
        system_deploy_blacklist=settings.system_paasta_config.
        get_deploy_blacklist())
    filtered_slaves = get_all_slaves_for_blacklist_whitelist(
        blacklist=monitoring_blacklist, whitelist=None)
    grouped_slaves = get_mesos_slaves_grouped_by_attribute(
        slaves=filtered_slaves, attribute=discover_location_type)

    # rebuild the dict, replacing the slave object with just their hostname
    slave_hostname_by_location = {
        attribute_value: [slave["hostname"] for slave in slaves]
        for attribute_value, slaves in grouped_slaves.items()
    }

    expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace(
        service, instance, settings.cluster)
    expected_count_per_location = int(expected_smartstack_count /
                                      len(slave_hostname_by_location))
    smartstack_status: MutableMapping[str, Any] = {
        "registration": registration,
        "expected_backends_per_location": expected_count_per_location,
        "locations": [],
    }

    for location, hosts in slave_hostname_by_location.items():
        synapse_host = hosts[0]
        sorted_backends = sorted(
            get_backends(
                registration,
                synapse_host=synapse_host,
                synapse_port=settings.system_paasta_config.get_synapse_port(),
                synapse_haproxy_url_format=settings.system_paasta_config.
                get_synapse_haproxy_url_format(),
            ),
            key=lambda backend: backend["status"],
            reverse=True,  # put 'UP' backends above 'MAINT' backends
        )
        matched_backends_and_tasks = match_backends_and_tasks(
            sorted_backends, tasks)
        location_dict = build_smartstack_location_dict(
            location, matched_backends_and_tasks,
            should_return_individual_backends)
        smartstack_status["locations"].append(location_dict)

    return smartstack_status
Exemple #8
0
def are_local_tasks_in_danger():
    try:
        system_paasta_config = utils.load_system_paasta_config()
        local_services = marathon_services_running_here()
        local_backends = get_backends(
            service=None,
            synapse_host=system_paasta_config.get_default_synapse_host(),
            synapse_port=system_paasta_config.get_synapse_port(),
            synapse_haproxy_url_format=system_paasta_config.get_synapse_haproxy_url_format()
        )
        for service, instance, port in local_services:
            log.info("Inspecting %s.%s on %s" % (service, instance, port))
            if is_healthy_in_haproxy(port, local_backends) and \
               synapse_replication_is_low(service, instance, system_paasta_config, local_backends=local_backends):
                log.warning("%s.%s on port %s is healthy but the service is in danger!" % (
                    service, instance, port
                ))
                return True
        return False
    except Exception:
        log.warning(traceback.format_exc())
        return False