def test_check_smartstack_replication_for_instance_crit_when_no_smartstack_info():
service = 'test'
instance = 'some_instance'
cluster = 'fake_cluster'
available = {}
expected_replication_count = 2
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_crit_when_absent():
service = 'test'
instance = 'some_absent_instance'
cluster = 'fake_cluster'
available = {'fake_region': {'test.two': 1, 'test.three': 4, 'test.four': 8}}
expected_replication_count = 8
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_ignores_things_under_a_different_namespace():
service = "test"
instance = "main"
namespace = "canary"
cluster = "fake_cluster"
available = {"fake_region": {"test.canary": 1, "test.main": 4, "test.fully_replicated": 8}}
expected_replication_count = 8
soa_dir = "test_dir"
fake_system_paasta_config = SystemPaastaConfig({}, "/fake/config")
crit = 90
with contextlib.nested(
mock.patch("paasta_tools.check_marathon_services_replication.send_event_if_under_replication", autospec=True),
mock.patch(
"paasta_tools.marathon_tools.read_registration_for_service_instance", autospec=True, return_value=namespace
),
mock.patch("paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service", autospec=True),
mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True),
) as (
mock_send_event_if_under_replication,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config
)
mock_send_event_if_under_replication.call_count == 0
def test_check_smartstack_replication_for_instance_ok_when_expecting_zero():
service = 'test'
instance = 'main'
cluster = 'fake_cluster'
available = {'fake_region': {'test.main': 1, 'test.three': 4, 'test.four': 8}}
expected_replication_count = 0
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_ok_with_enough_replication(
instance_config):
expected_replication_count = 8
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{
'fake_region': {
'test.canary': 1,
'test.low_replication': 4,
'fake_service.fake_instance': 8,
},
}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event:
check_marathon_services_replication.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
instance_config=instance_config,
status=pysensu_yelp.Status.OK,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert (
'{} has 8 out of 8 expected instances in fake_region (OK: 100%)'.
format(instance_config.job_id)) in alert_output
def test_check_smartstack_replication_for_instance_ignores_things_under_a_different_namespace():
service = 'test'
instance = 'main'
namespace = 'canary'
cluster = 'fake_cluster'
available = {'fake_region': {'test.canary': 1, 'test.main': 4, 'test.fully_replicated': 8}}
expected_replication_count = 8
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event_if_under_replication', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=namespace),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event_if_under_replication,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event_if_under_replication.call_count == 0
def test_check_smartstack_replication_for_instance_crit_when_missing_replication_multilocation(
instance_config, ):
expected_replication_count = 2
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{'fake_region': {'test.main': 0}, 'fake_other_region': {'test.main': 0}}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event:
check_marathon_services_replication.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
instance_config=instance_config,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert ("{} has 0 out of 1 expected instances in fake_region".format(
instance_config.job_id)) in alert_output
assert (
"{} has 0 out of 1 expected instances in fake_other_region".format(
instance_config.job_id)) in alert_output
def test_check_smartstack_replication_for_instance_crit_when_no_smartstack_info():
service = 'test'
instance = 'some_instance'
cluster = 'fake_cluster'
available = {}
expected_replication_count = 2
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_crit_when_low_replication(
instance_config):
expected_replication_count = 8
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{
'fake_region': {
'test.canary': 1,
'fake_service.fake_instance': 4,
'test.fully_replicated': 8,
},
}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event:
check_marathon_services_replication.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
instance_config=instance_config,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert ('Service {} has 4 out of 8 expected instances in fake_region'.
format(instance_config.job_id)) in alert_output
assert ('paasta status -s {} -i {} -c {} -vv'.format(
instance_config.service,
instance_config.instance,
instance_config.cluster,
)) in alert_output
def test_check_smartstack_replication_for_instance_ignores_things_under_a_different_namespace():
service = 'test'
instance = 'main'
namespace = 'canary'
cluster = 'fake_cluster'
available = {'fake_region': {'test.canary': 1, 'test.main': 4, 'test.fully_replicated': 8}}
expected_replication_count = 8
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event_if_under_replication', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=namespace),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event_if_under_replication,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event_if_under_replication.call_count == 0
def test_check_smartstack_replication_for_instance_ok_with_enough_replication_multilocation():
service = 'test'
instance = 'everything_up'
cluster = 'fake_cluster'
available = {'fake_region': {'test.everything_up': 1}, 'fake_other_region': {'test.everything_up': 1}}
expected_replication_count = 2
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_ok_when_expecting_zero():
service = 'test'
instance = 'main'
cluster = 'fake_cluster'
available = {'fake_region': {'test.main': 1, 'test.three': 4, 'test.four': 8}}
expected_replication_count = 0
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_crit_when_low_replication():
service = 'test'
instance = 'not_enough'
cluster = 'fake_cluster'
available = {'fake_region': {'test.canary': 1, 'test.not_enough': 4, 'test.fully_replicated': 8}}
expected_replication_count = 8
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_crit_when_absent():
service = 'test'
instance = 'some_absent_instance'
cluster = 'fake_cluster'
available = {'fake_region': {'test.two': 1, 'test.three': 4, 'test.four': 8}}
expected_replication_count = 8
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_crit_when_zero_replication(
):
service = 'test'
instance = 'zero_running'
cluster = 'fake_cluster'
available = {
'fake_region': {
'test.zero_running': 0,
'test.main': 8,
'test.fully_replicated': 8
}
}
expected_replication_count = 8
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance)),
mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True)) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert 'Service test.zero_running has 0 out of 8 expected instances in fake_region' in alert_output
assert "paasta status -s test -i zero_running -c fake_cluster -vv" in alert_output
def test_check_smartstack_replication_for_instance_ok_with_enough_replication_multilocation():
service = 'test'
instance = 'everything_up'
cluster = 'fake_cluster'
available = {'fake_region': {'test.everything_up': 1}, 'fake_other_region': {'test.everything_up': 1}}
expected_replication_count = 2
soa_dir = 'test_dir'
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, crit, expected_replication_count,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY)
def test_check_smartstack_replication_for_instance_ok_with_enough_replication(
):
service = 'test'
instance = 'everything_up'
cluster = 'fake_cluster'
available = {
'fake_region': {
'test.canary': 1,
'test.low_replication': 4,
'test.everything_up': 8
}
}
expected_replication_count = 8
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance)),
mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True)) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.everything_up has 8 out of 8 expected instances in fake_region (OK: 100%)" in alert_output
def test_check_smartstack_replication_for_instance_crit_when_low_replication_multilocation(
):
service = 'test'
instance = 'low_replication'
cluster = 'fake_cluster'
available = {
'fake_region': {
'test.low_replication': 1
},
'fake_other_region': {
'test.low_replication': 0
}
}
expected_replication_count = 2
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event, mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance),
), mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True,
) as mock_load_smartstack_info_for_service, mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True,
) as mock_load_marathon_service_config:
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.low_replication has 1 out of 1 expected instances in fake_region" in alert_output
assert "test.low_replication has 0 out of 1 expected instances in fake_other_region" in alert_output
assert "paasta status -s test -i low_replication -c fake_cluster -vv" in alert_output
def test_check_smartstack_replication_for_instance_ok_when_expecting_zero():
service = 'test'
instance = 'main'
cluster = 'fake_cluster'
available = {
'fake_region': {
'test.main': 1,
'test.three': 4,
'test.four': 8
}
}
expected_replication_count = 0
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event, mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance),
), mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True,
) as mock_load_smartstack_info_for_service, mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True,
) as mock_load_marathon_service_config:
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY,
)
def test_check_smartstack_replication_for_instance_crit_when_zero_replication_multilocation(
):
service = 'test'
instance = 'zero_running'
cluster = 'fake_cluster'
expected_replication_count = 2
soa_dir = 'test_dir'
crit = 90
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{
'fake_region': {'test.zero_running': 0}, 'fake_other_region':
{'test.zero_running': 0},
}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event, mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance),
), mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True,
) as mock_load_marathon_service_config:
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.zero_running has 0 out of 1 expected instances in fake_region (CRITICAL: 0%)" in alert_output
assert "test.zero_running has 0 out of 1 expected instances in fake_other_region (CRITICAL: 0%)" in alert_output
assert "paasta status -s test -i zero_running -c fake_cluster -vv" in alert_output
def test_check_smartstack_replication_for_instance_crit_when_no_smartstack_info(
):
service = 'test'
instance = 'some_instance'
cluster = 'fake_cluster'
available = {}
expected_replication_count = 2
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance)),
mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True),
mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True)) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count,
fake_system_paasta_config)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.some_instance has no Smartstack replication info." in alert_output
def test_check_smartstack_replication_for_instance_crit_when_low_replication():
service = 'test'
instance = 'not_enough'
cluster = 'fake_cluster'
expected_replication_count = 8
soa_dir = 'test_dir'
crit = 90
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{'fake_region': {'test.canary': 1, 'test.not_enough': 4, 'test.fully_replicated': 8}}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event, mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance),
), mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True,
) as mock_load_marathon_service_config:
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert 'Service test.not_enough has 4 out of 8 expected instances in fake_region' in alert_output
assert "paasta status -s test -i not_enough -c fake_cluster -vv" in alert_output
def test_check_smartstack_replication_for_instance_ignores_things_under_a_different_namespace(
):
service = 'test'
instance = 'main'
namespace = 'canary'
cluster = 'fake_cluster'
available = {
'fake_region': {
'test.canary': 1,
'test.main': 4,
'test.fully_replicated': 8
}
}
expected_replication_count = 8
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event_if_under_replication',
autospec=True,
) as mock_send_event_if_under_replication, mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=namespace,
), mock.patch(
'paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service',
autospec=True,
) as mock_load_smartstack_info_for_service, mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True,
) as mock_load_marathon_service_config:
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
fake_system_paasta_config,
)
mock_send_event_if_under_replication.call_count == 0
def test_check_smartstack_replication_for_instance_crit_when_absent(instance_config):
expected_replication_count = 8
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{'fake_region': {'test.two': 1, 'test.three': 4, 'test.four': 8}}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event', autospec=True,
) as mock_send_event:
check_marathon_services_replication.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
instance_config=instance_config,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
def test_check_smartstack_replication_for_instance_ignores_things_under_a_different_namespace(
instance_config,
):
instance_config.get_registrations.return_value = 'fake_registration'
expected_replication_count = 8
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{'fake_region': {'test.canary': 1, 'test.main': 4, 'test.fully_replicated': 8}}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event_if_under_replication',
autospec=True,
) as mock_send_event_if_under_replication:
check_marathon_services_replication.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event_if_under_replication.call_count == 0
def test_check_smartstack_replication_for_instance_crit_when_zero_replication_multilocation():
service = "test"
instance = "zero_running"
cluster = "fake_cluster"
available = {"fake_region": {"test.zero_running": 0}, "fake_other_region": {"test.zero_running": 0}}
expected_replication_count = 2
soa_dir = "test_dir"
fake_system_paasta_config = SystemPaastaConfig({}, "/fake/config")
crit = 90
with contextlib.nested(
mock.patch("paasta_tools.check_marathon_services_replication.send_event", autospec=True),
mock.patch(
"paasta_tools.marathon_tools.read_registration_for_service_instance",
autospec=True,
return_value=compose_job_id(service, instance),
),
mock.patch("paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service", autospec=True),
mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True),
) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.zero_running has 0 out of 1 expected instances in fake_region (CRITICAL: 0%)" in alert_output
assert "test.zero_running has 0 out of 1 expected instances in fake_other_region (CRITICAL: 0%)" in alert_output
assert "paasta status -s test -i zero_running -c fake_cluster -vv" in alert_output
def test_check_smartstack_replication_for_instance_ok_when_expecting_zero():
service = "test"
instance = "main"
cluster = "fake_cluster"
available = {"fake_region": {"test.main": 1, "test.three": 4, "test.four": 8}}
expected_replication_count = 0
soa_dir = "test_dir"
fake_system_paasta_config = SystemPaastaConfig({}, "/fake/config")
crit = 90
with contextlib.nested(
mock.patch("paasta_tools.check_marathon_services_replication.send_event", autospec=True),
mock.patch(
"paasta_tools.marathon_tools.read_registration_for_service_instance",
autospec=True,
return_value=compose_job_id(service, instance),
),
mock.patch("paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service", autospec=True),
mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True),
) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY,
)
def test_check_smartstack_replication_for_instance_ok_with_enough_replication():
service = "test"
instance = "everything_up"
cluster = "fake_cluster"
available = {"fake_region": {"test.canary": 1, "test.low_replication": 4, "test.everything_up": 8}}
expected_replication_count = 8
soa_dir = "test_dir"
fake_system_paasta_config = SystemPaastaConfig({}, "/fake/config")
crit = 90
with contextlib.nested(
mock.patch("paasta_tools.check_marathon_services_replication.send_event", autospec=True),
mock.patch(
"paasta_tools.marathon_tools.read_registration_for_service_instance",
autospec=True,
return_value=compose_job_id(service, instance),
),
mock.patch("paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service", autospec=True),
mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True),
) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.everything_up has 8 out of 8 expected instances in fake_region (OK: 100%)" in alert_output
def test_check_smartstack_replication_for_instance_crit_when_absent():
service = 'test'
instance = 'some_absent_instance'
cluster = 'fake_cluster'
expected_replication_count = 8
soa_dir = 'test_dir'
crit = 90
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = \
{'fake_region': {'test.two': 1, 'test.three': 4, 'test.four': 8}}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event, mock.patch(
'paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True,
return_value=compose_job_id(service, instance),
), mock.patch(
'paasta_tools.marathon_tools.load_marathon_service_config',
autospec=True,
) as mock_load_marathon_service_config:
mock_service_job_config = mock.MagicMock(
spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service,
instance,
cluster,
soa_dir,
expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
def test_check_smartstack_replication_for_instance_crit_when_low_replication_multilocation():
service = 'test'
instance = 'low_replication'
cluster = 'fake_cluster'
available = {'fake_region': {'test.low_replication': 1}, 'fake_other_region': {'test.low_replication': 0}}
expected_replication_count = 2
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_namespace_for_service_instance',
autospec=True, return_value=instance),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_namespace_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
mock_load_smartstack_info_for_service.return_value = available
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config,
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.low_replication has 1 out of 1 expected instances in fake_region" in alert_output
assert "test.low_replication has 0 out of 1 expected instances in fake_other_region" in alert_output
assert "paasta status -s test -i low_replication -c fake_cluster -vv" in alert_output
def test_check_smartstack_replication_for_instance_crit_when_no_smartstack_info():
service = 'test'
instance = 'some_instance'
cluster = 'fake_cluster'
available = {}
expected_replication_count = 2
soa_dir = 'test_dir'
fake_system_paasta_config = SystemPaastaConfig({}, '/fake/config')
crit = 90
with contextlib.nested(
mock.patch('paasta_tools.check_marathon_services_replication.send_event', autospec=True),
mock.patch('paasta_tools.marathon_tools.read_registration_for_service_instance',
autospec=True, return_value=compose_job_id(service, instance)),
mock.patch('paasta_tools.check_marathon_services_replication.load_smartstack_info_for_service', autospec=True),
mock.patch('paasta_tools.marathon_tools.load_marathon_service_config', autospec=True)
) as (
mock_send_event,
mock_read_registration_for_service_instance,
mock_load_smartstack_info_for_service,
mock_load_marathon_service_config,
):
mock_load_smartstack_info_for_service.return_value = available
mock_service_job_config = mock.MagicMock(spec_set=MarathonServiceConfig)
mock_service_job_config.get_replication_crit_percentage.return_value = crit
mock_load_marathon_service_config.return_value = mock_service_job_config
check_marathon_services_replication.check_smartstack_replication_for_instance(
service, instance, cluster, soa_dir, expected_replication_count, fake_system_paasta_config
)
mock_send_event.assert_called_once_with(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert "test.some_instance has no Smartstack replication info." in alert_output
def test_check_smartstack_replication_for_instance_crit_when_no_smartstack_info(
instance_config, ):
expected_replication_count = 2
mock_smartstack_replication_checker = mock.Mock()
mock_smartstack_replication_checker.get_replication_for_instance.return_value = {}
with mock.patch(
'paasta_tools.check_marathon_services_replication.send_event',
autospec=True,
) as mock_send_event:
check_marathon_services_replication.check_smartstack_replication_for_instance(
instance_config=instance_config,
expected_count=expected_replication_count,
smartstack_replication_checker=mock_smartstack_replication_checker,
)
mock_send_event.assert_called_once_with(
instance_config=instance_config,
status=pysensu_yelp.Status.CRITICAL,
output=mock.ANY,
)
_, send_event_kwargs = mock_send_event.call_args
alert_output = send_event_kwargs["output"]
assert ("{} has no Smartstack replication info.".format(
instance_config.job_id)) in alert_output
