def check_service_replication( instance_config, all_tasks, smartstack_replication_checker, ): """Checks a service's replication levels based on how the service's replication should be monitored. (smartstack or mesos) :param instance_config: an instance of MarathonServiceConfig :param smartstack_replication_checker: an instance of MesosSmartstackReplicationChecker """ expected_count = instance_config.get_instances() log.info("Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)) proxy_port = get_proxy_port_for_instance(instance_config) registrations = instance_config.get_registrations() # if the primary registration does not match the service_instance name then # the best we can do is check marathon for replication (for now). if proxy_port is not None and registrations[0] == instance_config.job_id: monitoring_tools.check_smartstack_replication_for_instance( instance_config=instance_config, expected_count=expected_count, smartstack_replication_checker=smartstack_replication_checker, ) else: check_healthy_marathon_tasks_for_service_instance( instance_config=instance_config, expected_count=expected_count, all_tasks=all_tasks, )
def check_kubernetes_pod_replication( instance_config: KubernetesDeploymentConfig, all_tasks_or_pods: Sequence[V1Pod], smartstack_replication_checker: KubeSmartstackReplicationChecker, ) -> Optional[bool]: """Checks a service's replication levels based on how the service's replication should be monitored. (smartstack or k8s) :param instance_config: an instance of KubernetesDeploymentConfig :param smartstack_replication_checker: an instance of KubeSmartstackReplicationChecker """ expected_count = instance_config.get_instances() log.info("Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)) proxy_port = get_proxy_port_for_instance(instance_config) registrations = instance_config.get_registrations() # if the primary registration does not match the service_instance name then # the best we can do is check k8s for replication (for now). if proxy_port is not None and registrations[0] == instance_config.job_id: is_well_replicated = monitoring_tools.check_smartstack_replication_for_instance( instance_config=instance_config, expected_count=expected_count, smartstack_replication_checker=smartstack_replication_checker, ) return is_well_replicated else: check_healthy_kubernetes_tasks_for_service_instance( instance_config=instance_config, expected_count=expected_count, all_pods=all_tasks_or_pods, ) return None
def check_service_replication( instance_config: MarathonServiceConfig, all_tasks_or_pods: Sequence[MarathonTask], replication_checker: MesosSmartstackEnvoyReplicationChecker, ) -> Optional[bool]: """Checks a service's replication levels based on how the service's replication should be monitored. (smartstack/envoy or mesos) :param instance_config: an instance of MarathonServiceConfig :param replication_checker: an instance of MesosSmartstackEnvoyReplicationChecker """ expected_count = instance_config.get_instances() log.info("Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)) proxy_port = get_proxy_port_for_instance(instance_config) registrations = instance_config.get_registrations() # if the primary registration does not match the service_instance name then # the best we can do is check marathon for replication (for now). if proxy_port is not None and registrations[0] == instance_config.job_id: is_well_replicated = monitoring_tools.check_replication_for_instance( instance_config=instance_config, expected_count=expected_count, replication_checker=replication_checker, ) return is_well_replicated else: check_healthy_marathon_tasks_for_service_instance( instance_config=instance_config, expected_count=expected_count, all_tasks=all_tasks_or_pods, ) return None
def check_kubernetes_pod_replication( instance_config: KubernetesDeploymentConfig, all_tasks_or_pods: Sequence[V1Pod], smartstack_replication_checker: KubeSmartstackReplicationChecker, default_alert_after: Optional[str] = DEFAULT_ALERT_AFTER, ) -> Optional[bool]: """Checks a service's replication levels based on how the service's replication should be monitored. (smartstack or k8s) :param instance_config: an instance of KubernetesDeploymentConfig :param smartstack_replication_checker: an instance of KubeSmartstackReplicationChecker """ expected_count = instance_config.get_instances() log.info( "Expecting %d total tasks for %s" % (expected_count, instance_config.job_id) ) proxy_port = get_proxy_port_for_instance(instance_config) registrations = instance_config.get_registrations() # If this instance does not autoscale and only has 1 instance, set alert after to 20m. # Otherwise, set it to 10 min. if ( not instance_config.is_autoscaling_enabled() and instance_config.get_instances() == 1 ): default_alert_after = "20m" if "monitoring" not in instance_config.config_dict: instance_config.config_dict["monitoring"] = {} instance_config.config_dict["monitoring"][ "alert_after" ] = instance_config.config_dict["monitoring"].get( "alert_after", default_alert_after ) # if the primary registration does not match the service_instance name then # the best we can do is check k8s for replication (for now). if proxy_port is not None and registrations[0] == instance_config.job_id: is_well_replicated = monitoring_tools.check_smartstack_replication_for_instance( instance_config=instance_config, expected_count=expected_count, smartstack_replication_checker=smartstack_replication_checker, ) return is_well_replicated else: check_healthy_kubernetes_tasks_for_service_instance( instance_config=instance_config, expected_count=expected_count, all_pods=all_tasks_or_pods, ) return None
def test_get_proxy_port_for_instance(): mock_config = mock.Mock( get_registrations=mock.Mock(return_value=["thing.main.sha.sha"]), soa_dir="/nail/blah", ) with mock.patch( "paasta_tools.long_running_service_tools.load_service_namespace_config", autospec=True, ) as mock_load_service_namespace_config: mock_load_service_namespace_config.return_value = {"proxy_port": 1234} assert (long_running_service_tools.get_proxy_port_for_instance( mock_config) == 1234) mock_load_service_namespace_config.assert_called_once_with( service="thing", namespace="main", soa_dir="/nail/blah")
def test_get_proxy_port_for_instance(): mock_config = mock.Mock( get_registrations=mock.Mock( return_value=['thing.main.sha.sha'], ), soa_dir='/nail/blah', ) with mock.patch( 'paasta_tools.long_running_service_tools.load_service_namespace_config', autospec=True, ) as mock_load_service_namespace_config: mock_load_service_namespace_config.return_value = { 'proxy_port': 1234, } assert long_running_service_tools.get_proxy_port_for_instance(mock_config) == 1234 mock_load_service_namespace_config.assert_called_once_with( service='thing', namespace='main', soa_dir='/nail/blah', )