def test_send_event_users_monitoring_tools_send_event_respects_alert_after():
fake_service_name = "superfast"
fake_namespace = "jellyfish"
fake_status = "999999"
fake_output = "YOU DID IT"
fake_cluster = "fake_cluster"
fake_monitoring_overrides = {"alert_after": "666m"}
fake_soa_dir = "/hi/hello/hey"
fake_cluster = "fake_cluster"
expected_check_name = "check_marathon_services_replication.%s" % compose_job_id(fake_service_name, fake_namespace)
with contextlib.nested(
mock.patch("paasta_tools.monitoring_tools.send_event", autospec=True),
mock.patch("paasta_tools.check_marathon_services_replication.load_system_paasta_config", autospec=True),
mock.patch("paasta_tools.check_marathon_services_replication._log", autospec=True),
mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True),
) as (send_event_patch, load_system_paasta_config_patch, log_patch, load_marathon_service_config_patch):
load_marathon_service_config_patch.return_value.get_monitoring.return_value = fake_monitoring_overrides
check_marathon_services_replication.send_event(
fake_service_name, fake_namespace, fake_cluster, fake_soa_dir, fake_status, fake_output
)
send_event_patch.call_count == 1
send_event_patch.assert_called_once_with(
fake_service_name, expected_check_name, mock.ANY, fake_status, fake_output, fake_soa_dir
)
# The overrides dictionary is mutated in the function under test, so
# we expect the send_event_patch to be called with something that is a
# superset of what we originally put in (fake_monitoring_overrides)
actual_overrides_used = send_event_patch.call_args[0][2]
assert set({"alert_after": "666m"}.items()).issubset(set(actual_overrides_used.items()))
assert not set({"alert_after": "2m"}.items()).issubset(set(actual_overrides_used.items()))
def test_send_event_users_monitoring_tools_send_event_respects_alert_after(instance_config):
fake_status = '999999'
fake_output = 'YOU DID IT'
instance_config.get_monitoring.return_value = {'alert_after': '666m'}
expected_check_name = (
'check_marathon_services_replication.%s' %
instance_config.job_id
)
with mock.patch(
"paasta_tools.monitoring_tools.send_event", autospec=True,
) as send_event_patch, mock.patch(
"paasta_tools.check_marathon_services_replication._log", autospec=True,
), mock.patch(
'paasta_tools.check_marathon_services_replication.monitoring_tools.get_runbook',
autospec=True,
return_value='y/runbook',
):
check_marathon_services_replication.send_event(
instance_config=instance_config,
status=fake_status,
output=fake_output,
)
send_event_patch.call_count == 1
send_event_patch.assert_called_once_with(
service=instance_config.service,
check_name=expected_check_name,
overrides={
'runbook': 'y/runbook',
'alert_after': '666m', 'check_every': '1m',
},
status=fake_status,
output=fake_output,
soa_dir=instance_config.soa_dir,
cluster=instance_config.cluster,
)
def test_send_event_users_monitoring_tools_send_event_respects_alert_after():
fake_service_name = 'superfast'
fake_namespace = 'jellyfish'
fake_status = '999999'
fake_output = 'YOU DID IT'
fake_cluster = 'fake_cluster'
fake_monitoring_overrides = {'alert_after': '666m'}
fake_soa_dir = '/hi/hello/hey'
fake_cluster = 'fake_cluster'
expected_check_name = 'check_marathon_services_replication.%s' % compose_job_id(
fake_service_name, fake_namespace)
with mock.patch(
"paasta_tools.monitoring_tools.send_event",
autospec=True,
) as send_event_patch, mock.patch(
'paasta_tools.check_marathon_services_replication.load_system_paasta_config',
autospec=True,
), mock.patch(
"paasta_tools.check_marathon_services_replication._log",
autospec=True,
), mock.patch(
"paasta_tools.marathon_tools.load_marathon_service_config",
autospec=True,
) as load_marathon_service_config_patch:
load_marathon_service_config_patch.return_value.get_monitoring.return_value = fake_monitoring_overrides
check_marathon_services_replication.send_event(
fake_service_name,
fake_namespace,
fake_cluster,
fake_soa_dir,
fake_status,
fake_output,
)
send_event_patch.call_count == 1
send_event_patch.assert_called_once_with(
fake_service_name,
expected_check_name,
mock.ANY,
fake_status,
fake_output,
fake_soa_dir,
cluster=fake_cluster,
)
# The overrides dictionary is mutated in the function under test, so
# we expect the send_event_patch to be called with something that is a
# superset of what we originally put in (fake_monitoring_overrides)
actual_overrides_used = send_event_patch.call_args[0][2]
assert set({
'alert_after': '666m'
}.items()).issubset(set(actual_overrides_used.items()))
assert not set({
'alert_after': '2m'
}.items()).issubset(set(actual_overrides_used.items()))
def test_send_event_users_monitoring_tools_send_event_properly():
fake_service_name = 'superfast'
fake_namespace = 'jellyfish'
fake_status = '999999'
fake_output = 'YOU DID IT'
fake_cluster = 'fake_cluster'
fake_monitoring_overrides = {'fake_key': 'fake_value'}
fake_soa_dir = '/hi/hello/hey'
fake_cluster = 'fake_cluster'
expected_check_name = 'check_marathon_services_replication.%s' % compose_job_id(fake_service_name, fake_namespace)
with contextlib.nested(
mock.patch("paasta_tools.monitoring_tools.send_event", autospec=True),
mock.patch('paasta_tools.check_marathon_services_replication.load_system_paasta_config', autospec=True),
mock.patch("paasta_tools.check_marathon_services_replication._log", autospec=True),
mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True),
) as (
send_event_patch,
load_system_paasta_config_patch,
log_patch,
load_marathon_service_config_patch,
):
load_marathon_service_config_patch.return_value.get_monitoring.return_value = fake_monitoring_overrides
check_marathon_services_replication.send_event(fake_service_name,
fake_namespace,
fake_cluster,
fake_soa_dir,
fake_status,
fake_output)
send_event_patch.assert_called_once_with(
fake_service_name,
expected_check_name,
mock.ANY,
fake_status,
fake_output,
fake_soa_dir
)
# The overrides dictionary is mutated in the function under test, so
# we expect the send_event_patch to be called with something that is a
# superset of what we originally put in (fake_monitoring_overrides)
actual_overrides_used = send_event_patch.call_args[0][2]
assert set({'alert_after': '2m'}.items()).issubset(set(actual_overrides_used.items()))
assert 'runbook' in actual_overrides_used
def delete_app(app_id, client, soa_dir):
"""Deletes a marathon app safely and logs to notify the user that it
happened"""
log.warn("%s appears to be old; attempting to delete" % app_id)
service, instance, _, __ = marathon_tools.deformat_job_id(app_id)
cluster = load_system_paasta_config().get_cluster()
try:
with bounce_lib.bounce_lock_zookeeper(marathon_tools.compose_job_id(service, instance)):
bounce_lib.delete_marathon_app(app_id, client)
send_event(
service=service,
namespace=instance,
cluster=cluster,
soa_dir=soa_dir,
status=pysensu_yelp.Status.OK,
output="This instance was removed and is no longer running",
)
log_line = "Deleted stale marathon job that looks lost: %s" % app_id
_log(service=service,
component='deploy',
level='event',
cluster=cluster,
instance=instance,
line=log_line)
except IOError:
log.debug("%s is being bounced, skipping" % app_id)
except Exception:
loglines = ['Exception raised during cleanup of service %s:' % service]
loglines.extend(traceback.format_exc().rstrip().split("\n"))
for logline in loglines:
_log(service=service,
component='deploy',
level='debug',
cluster=load_system_paasta_config().get_cluster(),
instance=instance,
line=logline)
raise
