def test_send_event_users_monitoring_tools_send_event_respects_alert_after(): fake_service_name = "superfast" fake_namespace = "jellyfish" fake_status = "999999" fake_output = "YOU DID IT" fake_cluster = "fake_cluster" fake_monitoring_overrides = {"alert_after": "666m"} fake_soa_dir = "/hi/hello/hey" fake_cluster = "fake_cluster" expected_check_name = "check_marathon_services_replication.%s" % compose_job_id(fake_service_name, fake_namespace) with contextlib.nested( mock.patch("paasta_tools.monitoring_tools.send_event", autospec=True), mock.patch("paasta_tools.check_marathon_services_replication.load_system_paasta_config", autospec=True), mock.patch("paasta_tools.check_marathon_services_replication._log", autospec=True), mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True), ) as (send_event_patch, load_system_paasta_config_patch, log_patch, load_marathon_service_config_patch): load_marathon_service_config_patch.return_value.get_monitoring.return_value = fake_monitoring_overrides check_marathon_services_replication.send_event( fake_service_name, fake_namespace, fake_cluster, fake_soa_dir, fake_status, fake_output ) send_event_patch.call_count == 1 send_event_patch.assert_called_once_with( fake_service_name, expected_check_name, mock.ANY, fake_status, fake_output, fake_soa_dir ) # The overrides dictionary is mutated in the function under test, so # we expect the send_event_patch to be called with something that is a # superset of what we originally put in (fake_monitoring_overrides) actual_overrides_used = send_event_patch.call_args[0][2] assert set({"alert_after": "666m"}.items()).issubset(set(actual_overrides_used.items())) assert not set({"alert_after": "2m"}.items()).issubset(set(actual_overrides_used.items()))
def test_send_event_users_monitoring_tools_send_event_respects_alert_after(instance_config): fake_status = '999999' fake_output = 'YOU DID IT' instance_config.get_monitoring.return_value = {'alert_after': '666m'} expected_check_name = ( 'check_marathon_services_replication.%s' % instance_config.job_id ) with mock.patch( "paasta_tools.monitoring_tools.send_event", autospec=True, ) as send_event_patch, mock.patch( "paasta_tools.check_marathon_services_replication._log", autospec=True, ), mock.patch( 'paasta_tools.check_marathon_services_replication.monitoring_tools.get_runbook', autospec=True, return_value='y/runbook', ): check_marathon_services_replication.send_event( instance_config=instance_config, status=fake_status, output=fake_output, ) send_event_patch.call_count == 1 send_event_patch.assert_called_once_with( service=instance_config.service, check_name=expected_check_name, overrides={ 'runbook': 'y/runbook', 'alert_after': '666m', 'check_every': '1m', }, status=fake_status, output=fake_output, soa_dir=instance_config.soa_dir, cluster=instance_config.cluster, )
def test_send_event_users_monitoring_tools_send_event_respects_alert_after(): fake_service_name = 'superfast' fake_namespace = 'jellyfish' fake_status = '999999' fake_output = 'YOU DID IT' fake_cluster = 'fake_cluster' fake_monitoring_overrides = {'alert_after': '666m'} fake_soa_dir = '/hi/hello/hey' fake_cluster = 'fake_cluster' expected_check_name = 'check_marathon_services_replication.%s' % compose_job_id( fake_service_name, fake_namespace) with mock.patch( "paasta_tools.monitoring_tools.send_event", autospec=True, ) as send_event_patch, mock.patch( 'paasta_tools.check_marathon_services_replication.load_system_paasta_config', autospec=True, ), mock.patch( "paasta_tools.check_marathon_services_replication._log", autospec=True, ), mock.patch( "paasta_tools.marathon_tools.load_marathon_service_config", autospec=True, ) as load_marathon_service_config_patch: load_marathon_service_config_patch.return_value.get_monitoring.return_value = fake_monitoring_overrides check_marathon_services_replication.send_event( fake_service_name, fake_namespace, fake_cluster, fake_soa_dir, fake_status, fake_output, ) send_event_patch.call_count == 1 send_event_patch.assert_called_once_with( fake_service_name, expected_check_name, mock.ANY, fake_status, fake_output, fake_soa_dir, cluster=fake_cluster, ) # The overrides dictionary is mutated in the function under test, so # we expect the send_event_patch to be called with something that is a # superset of what we originally put in (fake_monitoring_overrides) actual_overrides_used = send_event_patch.call_args[0][2] assert set({ 'alert_after': '666m' }.items()).issubset(set(actual_overrides_used.items())) assert not set({ 'alert_after': '2m' }.items()).issubset(set(actual_overrides_used.items()))
def test_send_event_users_monitoring_tools_send_event_properly(): fake_service_name = 'superfast' fake_namespace = 'jellyfish' fake_status = '999999' fake_output = 'YOU DID IT' fake_cluster = 'fake_cluster' fake_monitoring_overrides = {'fake_key': 'fake_value'} fake_soa_dir = '/hi/hello/hey' fake_cluster = 'fake_cluster' expected_check_name = 'check_marathon_services_replication.%s' % compose_job_id(fake_service_name, fake_namespace) with contextlib.nested( mock.patch("paasta_tools.monitoring_tools.send_event", autospec=True), mock.patch('paasta_tools.check_marathon_services_replication.load_system_paasta_config', autospec=True), mock.patch("paasta_tools.check_marathon_services_replication._log", autospec=True), mock.patch("paasta_tools.marathon_tools.load_marathon_service_config", autospec=True), ) as ( send_event_patch, load_system_paasta_config_patch, log_patch, load_marathon_service_config_patch, ): load_marathon_service_config_patch.return_value.get_monitoring.return_value = fake_monitoring_overrides check_marathon_services_replication.send_event(fake_service_name, fake_namespace, fake_cluster, fake_soa_dir, fake_status, fake_output) send_event_patch.assert_called_once_with( fake_service_name, expected_check_name, mock.ANY, fake_status, fake_output, fake_soa_dir ) # The overrides dictionary is mutated in the function under test, so # we expect the send_event_patch to be called with something that is a # superset of what we originally put in (fake_monitoring_overrides) actual_overrides_used = send_event_patch.call_args[0][2] assert set({'alert_after': '2m'}.items()).issubset(set(actual_overrides_used.items())) assert 'runbook' in actual_overrides_used
def delete_app(app_id, client, soa_dir): """Deletes a marathon app safely and logs to notify the user that it happened""" log.warn("%s appears to be old; attempting to delete" % app_id) service, instance, _, __ = marathon_tools.deformat_job_id(app_id) cluster = load_system_paasta_config().get_cluster() try: with bounce_lib.bounce_lock_zookeeper(marathon_tools.compose_job_id(service, instance)): bounce_lib.delete_marathon_app(app_id, client) send_event( service=service, namespace=instance, cluster=cluster, soa_dir=soa_dir, status=pysensu_yelp.Status.OK, output="This instance was removed and is no longer running", ) log_line = "Deleted stale marathon job that looks lost: %s" % app_id _log(service=service, component='deploy', level='event', cluster=cluster, instance=instance, line=log_line) except IOError: log.debug("%s is being bounced, skipping" % app_id) except Exception: loglines = ['Exception raised during cleanup of service %s:' % service] loglines.extend(traceback.format_exc().rstrip().split("\n")) for logline in loglines: _log(service=service, component='deploy', level='debug', cluster=load_system_paasta_config().get_cluster(), instance=instance, line=logline) raise