def check_service_replication(
instance_config,
all_tasks,
smartstack_replication_checker,
):
"""Checks a service's replication levels based on how the service's replication
should be monitored. (smartstack or mesos)
:param instance_config: an instance of MarathonServiceConfig
:param smartstack_replication_checker: an instance of MesosSmartstackReplicationChecker
"""
expected_count = instance_config.get_instances()
log.info("Expecting %d total tasks for %s" %
(expected_count, instance_config.job_id))
proxy_port = get_proxy_port_for_instance(instance_config)
registrations = instance_config.get_registrations()
# if the primary registration does not match the service_instance name then
# the best we can do is check marathon for replication (for now).
if proxy_port is not None and registrations[0] == instance_config.job_id:
monitoring_tools.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_count,
smartstack_replication_checker=smartstack_replication_checker,
)
else:
check_healthy_marathon_tasks_for_service_instance(
instance_config=instance_config,
expected_count=expected_count,
all_tasks=all_tasks,
)
def check_kubernetes_pod_replication(
instance_config: KubernetesDeploymentConfig,
all_tasks_or_pods: Sequence[V1Pod],
smartstack_replication_checker: KubeSmartstackReplicationChecker,
) -> Optional[bool]:
"""Checks a service's replication levels based on how the service's replication
should be monitored. (smartstack or k8s)
:param instance_config: an instance of KubernetesDeploymentConfig
:param smartstack_replication_checker: an instance of KubeSmartstackReplicationChecker
"""
expected_count = instance_config.get_instances()
log.info("Expecting %d total tasks for %s" %
(expected_count, instance_config.job_id))
proxy_port = get_proxy_port_for_instance(instance_config)
registrations = instance_config.get_registrations()
# if the primary registration does not match the service_instance name then
# the best we can do is check k8s for replication (for now).
if proxy_port is not None and registrations[0] == instance_config.job_id:
is_well_replicated = monitoring_tools.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_count,
smartstack_replication_checker=smartstack_replication_checker,
)
return is_well_replicated
else:
check_healthy_kubernetes_tasks_for_service_instance(
instance_config=instance_config,
expected_count=expected_count,
all_pods=all_tasks_or_pods,
)
return None
def check_service_replication(
instance_config: MarathonServiceConfig,
all_tasks_or_pods: Sequence[MarathonTask],
replication_checker: MesosSmartstackEnvoyReplicationChecker,
) -> Optional[bool]:
"""Checks a service's replication levels based on how the service's replication
should be monitored. (smartstack/envoy or mesos)
:param instance_config: an instance of MarathonServiceConfig
:param replication_checker: an instance of MesosSmartstackEnvoyReplicationChecker
"""
expected_count = instance_config.get_instances()
log.info("Expecting %d total tasks for %s" %
(expected_count, instance_config.job_id))
proxy_port = get_proxy_port_for_instance(instance_config)
registrations = instance_config.get_registrations()
# if the primary registration does not match the service_instance name then
# the best we can do is check marathon for replication (for now).
if proxy_port is not None and registrations[0] == instance_config.job_id:
is_well_replicated = monitoring_tools.check_replication_for_instance(
instance_config=instance_config,
expected_count=expected_count,
replication_checker=replication_checker,
)
return is_well_replicated
else:
check_healthy_marathon_tasks_for_service_instance(
instance_config=instance_config,
expected_count=expected_count,
all_tasks=all_tasks_or_pods,
)
return None
def check_kubernetes_pod_replication(
instance_config: KubernetesDeploymentConfig,
all_tasks_or_pods: Sequence[V1Pod],
smartstack_replication_checker: KubeSmartstackReplicationChecker,
default_alert_after: Optional[str] = DEFAULT_ALERT_AFTER,
) -> Optional[bool]:
"""Checks a service's replication levels based on how the service's replication
should be monitored. (smartstack or k8s)
:param instance_config: an instance of KubernetesDeploymentConfig
:param smartstack_replication_checker: an instance of KubeSmartstackReplicationChecker
"""
expected_count = instance_config.get_instances()
log.info(
"Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)
)
proxy_port = get_proxy_port_for_instance(instance_config)
registrations = instance_config.get_registrations()
# If this instance does not autoscale and only has 1 instance, set alert after to 20m.
# Otherwise, set it to 10 min.
if (
not instance_config.is_autoscaling_enabled()
and instance_config.get_instances() == 1
):
default_alert_after = "20m"
if "monitoring" not in instance_config.config_dict:
instance_config.config_dict["monitoring"] = {}
instance_config.config_dict["monitoring"][
"alert_after"
] = instance_config.config_dict["monitoring"].get(
"alert_after", default_alert_after
)
# if the primary registration does not match the service_instance name then
# the best we can do is check k8s for replication (for now).
if proxy_port is not None and registrations[0] == instance_config.job_id:
is_well_replicated = monitoring_tools.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_count,
smartstack_replication_checker=smartstack_replication_checker,
)
return is_well_replicated
else:
check_healthy_kubernetes_tasks_for_service_instance(
instance_config=instance_config,
expected_count=expected_count,
all_pods=all_tasks_or_pods,
)
return None
def test_get_proxy_port_for_instance():
mock_config = mock.Mock(
get_registrations=mock.Mock(return_value=["thing.main.sha.sha"]),
soa_dir="/nail/blah",
)
with mock.patch(
"paasta_tools.long_running_service_tools.load_service_namespace_config",
autospec=True,
) as mock_load_service_namespace_config:
mock_load_service_namespace_config.return_value = {"proxy_port": 1234}
assert (long_running_service_tools.get_proxy_port_for_instance(
mock_config) == 1234)
mock_load_service_namespace_config.assert_called_once_with(
service="thing", namespace="main", soa_dir="/nail/blah")
def test_get_proxy_port_for_instance():
mock_config = mock.Mock(
get_registrations=mock.Mock(
return_value=['thing.main.sha.sha'],
),
soa_dir='/nail/blah',
)
with mock.patch(
'paasta_tools.long_running_service_tools.load_service_namespace_config', autospec=True,
) as mock_load_service_namespace_config:
mock_load_service_namespace_config.return_value = {
'proxy_port': 1234,
}
assert long_running_service_tools.get_proxy_port_for_instance(mock_config) == 1234
mock_load_service_namespace_config.assert_called_once_with(
service='thing',
namespace='main',
soa_dir='/nail/blah',
)