class CreateIncidentTest(TestCase): record_event = patcher("sentry.analytics.base.Analytics.record_event") calculate_incident_suspects = patcher("sentry.incidents.tasks.calculate_incident_suspects") def test_simple(self): incident_type = IncidentType.CREATED title = "hello" query = "goodbye" date_started = timezone.now() other_project = self.create_project() other_group = self.create_group(project=other_project) self.record_event.reset_mock() incident = create_incident( self.organization, type=incident_type, title=title, query=query, date_started=date_started, projects=[self.project], groups=[self.group, other_group], ) assert incident.identifier == 1 assert incident.status == incident_type.value assert incident.title == title assert incident.query == query assert incident.date_started == date_started assert incident.date_detected == date_started assert ( IncidentGroup.objects.filter( incident=incident, group__in=[self.group, other_group] ).count() == 2 ) assert ( IncidentProject.objects.filter( incident=incident, project__in=[self.project, other_project] ).count() == 2 ) assert ( IncidentActivity.objects.filter( incident=incident, type=IncidentActivityType.CREATED.value, event_stats_snapshot__isnull=False, ).count() == 1 ) assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentCreatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(IncidentType.CREATED.value), } self.calculate_incident_suspects.apply_async.assert_called_once_with( kwargs={"incident_id": incident.id} )
class UpdateIncidentStatus(TestCase): record_event = patcher("sentry.analytics.base.Analytics.record_event") def get_most_recent_incident_activity(self, incident): return IncidentActivity.objects.filter(incident=incident).order_by("-id")[:1].get() def test_status_already_set(self): incident = self.create_incident(status=IncidentStatus.WARNING.value) update_incident_status(incident, IncidentStatus.WARNING) assert incident.status == IncidentStatus.WARNING.value def run_test(self, incident, status, expected_date_closed, user=None, comment=None): prev_status = incident.status self.record_event.reset_mock() update_incident_status(incident, status, user=user, comment=comment) incident = Incident.objects.get(id=incident.id) assert incident.status == status.value assert incident.date_closed == expected_date_closed activity = self.get_most_recent_incident_activity(incident) assert activity.type == IncidentActivityType.STATUS_CHANGE.value assert activity.user == user if user: assert IncidentSubscription.objects.filter(incident=incident, user=user).exists() assert activity.value == six.text_type(status.value) assert activity.previous_value == six.text_type(prev_status) assert activity.comment == comment assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentStatusUpdatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(incident.type), "prev_status": six.text_type(prev_status), "status": six.text_type(incident.status), } def test_closed(self): incident = create_incident( self.organization, IncidentType.ALERT_TRIGGERED, "Test", "", QueryAggregations.TOTAL, timezone.now(), projects=[self.project], ) with self.assertChanges( lambda: IncidentSnapshot.objects.filter(incident=incident).exists(), before=False, after=True, ): self.run_test(incident, IncidentStatus.CLOSED, timezone.now()) def test_all_params(self): incident = self.create_incident() self.run_test( incident, IncidentStatus.CLOSED, timezone.now(), user=self.user, comment="lol" )
class BaseSnubaTaskTest(metaclass=abc.ABCMeta): metrics = patcher("sentry.snuba.tasks.metrics") status_translations = { QuerySubscription.Status.CREATING: "create", QuerySubscription.Status.UPDATING: "update", QuerySubscription.Status.DELETING: "delete", } @abc.abstractproperty def expected_status(self): pass @abc.abstractmethod def task(self): pass def create_subscription(self, status=None, subscription_id=None, dataset=None, query=None): if status is None: status = self.expected_status if dataset is None: dataset = QueryDatasets.EVENTS dataset = dataset.value aggregate = "count_unique(tags[sentry:user])" if query is None: query = "hello" time_window = 60 resolution = 60 snuba_query = SnubaQuery.objects.create( dataset=dataset, aggregate=aggregate, query=query, time_window=time_window, resolution=resolution, ) return QuerySubscription.objects.create( snuba_query=snuba_query, status=status.value, subscription_id=subscription_id, project=self.project, type="something", ) def test_no_subscription(self): self.task(12345) self.metrics.incr.assert_called_once_with( "snuba.subscriptions.{}.subscription_does_not_exist".format( self.status_translations[self.expected_status])) def test_invalid_status(self): sub = self.create_subscription(QuerySubscription.Status.ACTIVE) self.task(sub.id) self.metrics.incr.assert_called_once_with( "snuba.subscriptions.{}.incorrect_status".format( self.status_translations[self.expected_status]))
class TestSendSubscriberNotifications(BaseIncidentActivityTest, TestCase): send_async = patcher("sentry.utils.email.MessageBuilder.send_async") def test_simple(self): activity = create_incident_activity( self.incident, IncidentActivityType.COMMENT, user=self.user, comment="hello" ) send_subscriber_notifications(activity.id) # User shouldn't receive an email for their own activity self.send_async.assert_not_called() # NOQA self.send_async.reset_mock() non_member_user = self.create_user(email="*****@*****.**") subscribe_to_incident(activity.incident, non_member_user) member_user = self.create_user(email="*****@*****.**") self.create_member([self.team], user=member_user, organization=self.organization) subscribe_to_incident(activity.incident, member_user) send_subscriber_notifications(activity.id) self.send_async.assert_called_once_with([member_user.email]) assert not IncidentSubscription.objects.filter( incident=activity.incident, user=non_member_user ).exists() assert IncidentSubscription.objects.filter( incident=activity.incident, user=member_user ).exists() def test_invalid_types(self): activity_type = IncidentActivityType.CREATED activity = create_incident_activity(self.incident, activity_type) send_subscriber_notifications(activity.id) self.send_async.assert_not_called() # NOQA self.send_async.reset_mock()
class AlertRuleTriggerActionActivateTest(TestCase): metrics = patcher("sentry.incidents.models.metrics") def setUp(self): self.old_handlers = AlertRuleTriggerAction._type_registrations AlertRuleTriggerAction._type_registrations = {} def tearDown(self): AlertRuleTriggerAction._type_registrations = self.old_handlers def test_unhandled(self): trigger = AlertRuleTriggerAction( type=AlertRuleTriggerAction.Type.EMAIL.value) trigger.build_handler(Mock(), Mock()) self.metrics.incr.assert_called_once_with( "alert_rule_trigger.unhandled_type.0") def test_handled(self): mock_handler = Mock() type = AlertRuleTriggerAction.Type.EMAIL AlertRuleTriggerAction.register_type("something", type, [])(mock_handler) trigger = AlertRuleTriggerAction( type=AlertRuleTriggerAction.Type.EMAIL.value) incident = Mock() project = Mock() trigger.build_handler(incident, project) mock_handler.assert_called_once_with(trigger, incident, project) assert not self.metrics.incr.called
class HandleMessageTest(BaseQuerySubscriptionTest, TestCase): metrics = patcher("sentry.snuba.query_subscription_consumer.metrics") def test_no_subscription(self): with mock.patch("sentry.snuba.tasks._snuba_pool") as pool: pool.urlopen.return_value.status = 202 self.consumer.handle_message( self.build_mock_message( self.valid_wrapper, topic=settings.KAFKA_METRICS_SUBSCRIPTIONS_RESULTS ) ) pool.urlopen.assert_called_once_with( "DELETE", "/{}/{}/subscriptions/{}".format( QueryDatasets.METRICS.value, EntityKey.MetricsCounters.value, self.valid_payload["subscription_id"], ), ) self.metrics.incr.assert_called_once_with( "snuba_query_subscriber.subscription_doesnt_exist" ) def test_subscription_not_registered(self): sub = QuerySubscription.objects.create( project=self.project, type="unregistered", subscription_id="an_id" ) data = self.valid_wrapper data["payload"]["subscription_id"] = sub.subscription_id self.consumer.handle_message(self.build_mock_message(data)) self.metrics.incr.assert_called_once_with( "snuba_query_subscriber.subscription_type_not_registered" ) def test_subscription_registered(self): registration_key = "registered_test" mock_callback = mock.Mock() register_subscriber(registration_key)(mock_callback) with self.tasks(): snuba_query = create_snuba_query( QueryDatasets.EVENTS, "hello", "count()", timedelta(minutes=10), timedelta(minutes=1), None, ) sub = create_snuba_subscription(self.project, registration_key, snuba_query) sub.refresh_from_db() data = self.valid_wrapper data["payload"]["subscription_id"] = sub.subscription_id self.consumer.handle_message(self.build_mock_message(data)) data = deepcopy(data) data["payload"]["values"] = data["payload"]["result"] data["payload"]["timestamp"] = parse_date(data["payload"]["timestamp"]).replace( tzinfo=pytz.utc ) mock_callback.assert_called_once_with(data["payload"], sub)
class CreateIncidentTest(TestCase): record_event = patcher("sentry.analytics.base.Analytics.record_event") def test_simple(self): incident_type = IncidentType.ALERT_TRIGGERED title = "hello" query = "goodbye" aggregation = QueryAggregations.UNIQUE_USERS date_started = timezone.now() other_project = self.create_project(fire_project_created=True) other_group = self.create_group(project=other_project) alert_rule = create_alert_rule( self.organization, [self.project], "hello", "level:error", QueryAggregations.TOTAL, 10, 1, ) self.record_event.reset_mock() incident = create_incident( self.organization, type=incident_type, title=title, query=query, aggregation=aggregation, date_started=date_started, projects=[self.project], groups=[self.group, other_group], alert_rule=alert_rule, ) assert incident.identifier == 1 assert incident.status == IncidentStatus.OPEN.value assert incident.type == incident_type.value assert incident.title == title assert incident.query == query assert incident.aggregation == aggregation.value assert incident.date_started == date_started assert incident.date_detected == date_started assert incident.alert_rule == alert_rule assert (IncidentGroup.objects.filter( incident=incident, group__in=[self.group, other_group]).count() == 2) assert (IncidentProject.objects.filter( incident=incident, project__in=[self.project, other_project]).count() == 2) assert (IncidentActivity.objects.filter( incident=incident, type=IncidentActivityType.DETECTED.value).count() == 1) assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentCreatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(IncidentType.ALERT_TRIGGERED.value), }
class HandleMessageTest(BaseQuerySubscriptionTest, TestCase): metrics = patcher("sentry.snuba.query_subscription_consumer.metrics") def test_no_subscription(self): with mock.patch("sentry.snuba.tasks._snuba_pool") as pool: pool.urlopen.return_value.status = 202 self.consumer.handle_message( self.build_mock_message( self.valid_wrapper, topic=settings.KAFKA_EVENTS_SUBSCRIPTIONS_RESULTS)) pool.urlopen.assert_called_once_with( "DELETE", "/{}/subscriptions/{}".format( QueryDatasets.EVENTS.value, self.valid_payload["subscription_id"]), ) self.metrics.incr.assert_called_once_with( "snuba_query_subscriber.subscription_doesnt_exist") def test_subscription_not_registered(self): sub = QuerySubscription.objects.create( project=self.project, type="unregistered", subscription_id="an_id", dataset="something", query="hello", aggregation=0, time_window=1, resolution=1, ) data = self.valid_wrapper data["payload"]["subscription_id"] = sub.subscription_id self.consumer.handle_message(self.build_mock_message(data)) self.metrics.incr.assert_called_once_with( "snuba_query_subscriber.subscription_type_not_registered") def test_subscription_registered(self): registration_key = "registered_test" mock_callback = Mock() register_subscriber(registration_key)(mock_callback) sub = QuerySubscription.objects.create( project=self.project, type=registration_key, subscription_id="an_id", dataset="something", query="hello", aggregation=0, time_window=1, resolution=1, ) data = self.valid_wrapper data["payload"]["subscription_id"] = sub.subscription_id self.consumer.handle_message(self.build_mock_message(data)) data = deepcopy(data) data["payload"]["values"] = data["payload"]["result"] data["payload"]["timestamp"] = parse_date( data["payload"]["timestamp"]).replace(tzinfo=pytz.utc) mock_callback.assert_called_once_with(data["payload"], sub)
class HandleTriggerActionTest(TestCase): metrics = patcher("sentry.incidents.tasks.metrics") @fixture def alert_rule(self): return self.create_alert_rule() @fixture def trigger(self): return create_alert_rule_trigger(self.alert_rule, CRITICAL_TRIGGER_LABEL, 100) @fixture def action(self): return create_alert_rule_trigger_action( self.trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER) def test_missing_trigger_action(self): with self.tasks(): handle_trigger_action.delay(1000, 1001, self.project.id, "hello") self.metrics.incr.assert_called_once_with( "incidents.alert_rules.action.skipping_missing_action") def test_missing_incident(self): with self.tasks(): handle_trigger_action.delay(self.action.id, 1001, self.project.id, "hello") self.metrics.incr.assert_called_once_with( "incidents.alert_rules.action.skipping_missing_incident") def test_missing_project(self): incident = self.create_incident() with self.tasks(): handle_trigger_action.delay(self.action.id, incident.id, 1002, "hello") self.metrics.incr.assert_called_once_with( "incidents.alert_rules.action.skipping_missing_project") def test(self): with patch.object(AlertRuleTriggerAction, "_type_registrations", new={}): mock_handler = Mock() AlertRuleTriggerAction.register_type( "email", AlertRuleTriggerAction.Type.EMAIL, [])(mock_handler) incident = self.create_incident() metric_value = 1234 with self.tasks(): handle_trigger_action.delay(self.action.id, incident.id, self.project.id, "fire", metric_value=metric_value) mock_handler.assert_called_once_with(self.action, incident, self.project) mock_handler.return_value.fire.assert_called_once_with( metric_value, IncidentStatus.CRITICAL)
class HandleTriggerActionTest(TestCase): metrics = patcher("sentry.incidents.tasks.metrics") @fixture def alert_rule(self): return self.create_alert_rule() @fixture def trigger(self): return create_alert_rule_trigger(self.alert_rule, "", AlertRuleThresholdType.ABOVE, 100) @fixture def action(self): return create_alert_rule_trigger_action( self.trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER) def test_missing_trigger_action(self): with self.tasks(): handle_trigger_action.delay(1000, 1001, self.project.id, "hello") self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_missing_action") def test_missing_incident(self): with self.tasks(): handle_trigger_action.delay(self.action.id, 1001, self.project.id, "hello") self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_missing_incident") def test_missing_project(self): incident = self.create_incident() with self.tasks(): handle_trigger_action.delay(self.action.id, incident.id, 1002, "hello") self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_missing_project") def test(self): with patch.object(AlertRuleTriggerAction, "handlers", new={}): mock_handler = Mock() AlertRuleTriggerAction.register_type_handler( AlertRuleTriggerAction.Type.EMAIL)(mock_handler) incident = self.create_incident() with self.tasks(): handle_trigger_action.delay(self.action.id, incident.id, self.project.id, "fire") mock_handler.assert_called_once_with(self.action, incident, self.project) mock_handler.return_value.fire.assert_called_once_with()
class HandleMessageTest(BaseQuerySubscriptionTest, TestCase): metrics = patcher("sentry.snuba.query_subscription_consumer.metrics") def test_no_subscription(self): self.consumer.handle_message(self.build_mock_message(self.valid_wrapper)) self.metrics.incr.assert_called_once_with( "snuba_query_subscriber.subscription_doesnt_exist" ) def test_subscription_not_registered(self): sub = QuerySubscription.objects.create( project=self.project, type="unregistered", subscription_id="an_id", dataset="something", query="hello", aggregation=0, time_window=1, resolution=1, ) data = self.valid_wrapper data["payload"]["subscription_id"] = sub.subscription_id self.consumer.handle_message(self.build_mock_message(data)) self.metrics.incr.assert_called_once_with( "snuba_query_subscriber.subscription_type_not_registered" ) def test_subscription_registered(self): registration_key = "registered_test" mock_callback = Mock() register_subscriber(registration_key)(mock_callback) sub = QuerySubscription.objects.create( project=self.project, type=registration_key, subscription_id="an_id", dataset="something", query="hello", aggregation=0, time_window=1, resolution=1, ) data = self.valid_wrapper data["payload"]["subscription_id"] = sub.subscription_id self.consumer.handle_message(self.build_mock_message(data)) data["payload"]["timestamp"] = parse_date(data["payload"]["timestamp"]).replace( tzinfo=pytz.utc ) mock_callback.assert_called_once_with(data["payload"], sub)
class BaseSnubaTaskTest(object): metrics = patcher("sentry.snuba.tasks.metrics") status_translations = { QuerySubscription.Status.CREATING: "create", QuerySubscription.Status.UPDATING: "update", QuerySubscription.Status.DELETING: "delete", } @abc.abstractproperty def expected_status(self): pass @abc.abstractmethod def task(self): pass def create_subscription(self, status=None, subscription_id=None): if status is None: status = self.expected_status return QuerySubscription.objects.create( status=status.value, subscription_id=subscription_id, project=self.project, type="something", dataset=QueryDatasets.EVENTS.value, query="hello", aggregation=QueryAggregations.UNIQUE_USERS.value, time_window=60, resolution=60, ) def test_no_subscription(self): self.task(12345) self.metrics.incr.assert_called_once_with( "snuba.subscriptions.{}.subscription_does_not_exist".format( self.status_translations[self.expected_status] ) ) def test_invalid_status(self): sub = self.create_subscription(QuerySubscription.Status.ACTIVE) self.task(sub.id) self.metrics.incr.assert_called_once_with( "snuba.subscriptions.{}.incorrect_status".format( self.status_translations[self.expected_status] ) )
class SwitchFormTest(Exam, unittest2.TestCase): mock_switch = fixture(Mock, conditions=[1, 2, 3]) condition_form = patcher('gutter.django.forms.ConditionForm') form = fixture(SwitchForm) @fixture def switch_from_object(self): return SwitchForm.from_object(self.mock_switch) @patch('gutter.django.forms.ConditionFormSet') def test_from_object_returns_dict_of_properties(self, _): eq_( self.switch_from_object.initial, dict( label=self.mock_switch.label, name=self.mock_switch.name, description=self.mock_switch.description, state=self.mock_switch.state, compounded=self.mock_switch.compounded, concent=self.mock_switch.concent, )) @patch('gutter.django.forms.ConditionFormSet') def test_from_object_sets_conditions_as_form_set(self, ConditionFormSet): eq_(self.switch_from_object.conditions, ConditionFormSet.return_value) # Called with a map() over ConditionForm.to_dict expected = [self.condition_form.to_dict.return_value] * 3 ConditionFormSet.assert_called_once_with(initial=expected) # Assert that the calls it did receive are correct self.condition_form.to_dict.assert_any_call(1) self.condition_form.to_dict.assert_any_call(2) self.condition_form.to_dict.assert_any_call(3) def test_from_object_marks_the_name_field_as_readonly(self): self.assertTrue( self.switch_from_object.fields['name'].widget.attrs['readonly'])
class ProcessUpdateTest(TestCase): metrics = patcher("sentry.incidents.subscription_processor.metrics") @fixture def other_project(self): return self.create_project() @fixture def sub(self): return self.rule.query_subscriptions.filter(project=self.project).get() @fixture def other_sub(self): return self.rule.query_subscriptions.filter(project=self.other_project).get() @fixture def rule(self): rule = create_alert_rule( self.organization, [self.project, self.other_project], "some rule", AlertRuleThresholdType.ABOVE, query="", aggregation=QueryAggregations.TOTAL, time_window=1, alert_threshold=100, resolve_threshold=10, threshold_period=1, ) return rule def build_subscription_update(self, subscription, time_delta=None, value=None): if time_delta is not None: timestamp = int(to_timestamp(timezone.now() + time_delta)) else: timestamp = int(time()) values = {} if subscription: aggregation_type = query_aggregation_to_snuba[ QueryAggregations(subscription.aggregation) ] value = randint(0, 100) if value is None else value values = {aggregation_type[2]: value} return { "subscription_id": subscription.subscription_id if subscription else uuid4().hex, "values": values, "timestamp": timestamp, "interval": 1, "partition": 1, "offset": 1, } def send_update(self, rule, value, time_delta=None, subscription=None): if time_delta is None: time_delta = timedelta() if subscription is None: subscription = self.sub processor = SubscriptionProcessor(subscription) message = self.build_subscription_update(subscription, value=value, time_delta=time_delta) processor.process_update(message) return processor def assert_no_active_incident(self, rule, subscription=None): assert not self.active_incident_exists(rule, subscription=subscription) def assert_active_incident(self, rule, subscription=None): assert self.active_incident_exists(rule, subscription=subscription) def active_incident_exists(self, rule, subscription=None): if subscription is None: subscription = self.sub return Incident.objects.filter( type=IncidentType.ALERT_TRIGGERED.value, status=IncidentStatus.OPEN.value, alert_rule=rule, projects=subscription.project, ).exists() def assert_trigger_counts(self, processor, alert_triggers=0, resolve_triggers=0): assert processor.alert_triggers == alert_triggers assert processor.resolve_triggers == resolve_triggers assert get_alert_rule_stats(processor.alert_rule, processor.subscription)[1:] == ( alert_triggers, resolve_triggers, ) def test_removed_alert_rule(self): message = self.build_subscription_update(self.sub) self.rule.delete() SubscriptionProcessor(self.sub).process_update(message) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.no_alert_rule_for_subscription" ) # TODO: Check subscription is deleted once we start doing that def test_skip_already_processed_update(self): self.send_update(self.rule, self.rule.alert_threshold) self.metrics.incr.reset_mock() self.send_update(self.rule, self.rule.alert_threshold) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update" ) self.metrics.incr.reset_mock() self.send_update(self.rule, self.rule.alert_threshold, timedelta(hours=-1)) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update" ) self.metrics.incr.reset_mock() self.send_update(self.rule, self.rule.alert_threshold, timedelta(hours=1)) self.metrics.incr.assert_not_called() # NOQA def test_no_alert(self): rule = self.rule processor = self.send_update(rule, rule.alert_threshold) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(self.rule) def test_alert(self): # Verify that an alert rule that only expects a single update to be over the # alert threshold triggers correctly rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) def test_alert_multiple_triggers(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold triggers correctly rule = self.rule rule.update(threshold_period=2) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) def test_alert_multiple_triggers_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold doesn't trigger if there are two updates that are above with # an update that is below the threshold in the middle rule = self.rule rule.update(threshold_period=2) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule) def test_no_active_incident_resolve(self): # Test that we don't track stats for resolving if there are no active incidents # related to the alert rule. rule = self.rule processor = self.send_update(rule, rule.resolve_threshold - 1) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_resolve(self): # Verify that an alert rule that only expects a single update to be under the # resolve threshold triggers correctly rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_resolve_multiple_triggers(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold triggers correctly rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) rule.update(threshold_period=2) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_resolve_multiple_triggers_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold doesn't trigger if there's two updates that are below with # an update that is above the threshold in the middle rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-4)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) rule.update(threshold_period=2) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule) def test_reversed_threshold_alert(self): # Test that inverting thresholds correctly alerts rule = self.rule rule.update(threshold_type=AlertRuleThresholdType.BELOW.value) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) def test_reversed_threshold_resolve(self): # Test that inverting thresholds correctly resolves rule = self.rule rule.update(threshold_type=AlertRuleThresholdType.BELOW.value) processor = self.send_update(rule, rule.alert_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_multiple_subscriptions_do_not_conflict(self): # Verify that multiple subscriptions associated with a rule don't conflict with # each other rule = self.rule rule.update(threshold_period=2) # Send an update through for the first subscription. This shouldn't trigger an # incident, since we need two consecutive updates that are over the threshold. processor = self.send_update( rule, rule.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule, self.sub) # Have an update come through for the other sub. This shouldn't influence the original processor = self.send_update( rule, rule.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub ) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule, self.sub) self.assert_no_active_incident(rule, self.other_sub) # Send another update through for the first subscription. This should trigger an # incident for just this subscription. processor = self.send_update( rule, rule.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule, self.sub) self.assert_no_active_incident(rule, self.other_sub) # Send another update through for the second subscription. This should trigger an # incident for just this subscription. processor = self.send_update( rule, rule.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub ) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule, self.sub) self.assert_active_incident(rule, self.other_sub) # Now we want to test that resolving is isolated. Send another update through # for the first subscription. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub ) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule, self.sub) self.assert_active_incident(rule, self.other_sub) processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub ) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule, self.sub) self.assert_active_incident(rule, self.other_sub) # This second update for the second subscription should resolve its incident, # but not the incident from the first subscription. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub ) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule, self.sub) self.assert_no_active_incident(rule, self.other_sub) # This second update for the first subscription should resolve its incident now. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub ) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_no_active_incident(rule, self.other_sub)
class CreateIncidentActivityTest(TestCase, BaseIncidentsTest): send_subscriber_notifications = patcher( "sentry.incidents.tasks.send_subscriber_notifications") record_event = patcher("sentry.analytics.base.Analytics.record_event") def assert_notifications_sent(self, activity): self.send_subscriber_notifications.apply_async.assert_called_once_with( kwargs={"activity_id": activity.id}, countdown=10) def test_no_snapshot(self): incident = self.create_incident() self.record_event.reset_mock() activity = create_incident_activity( incident, IncidentActivityType.STATUS_CHANGE, user=self.user, value=six.text_type(IncidentStatus.CLOSED.value), previous_value=six.text_type(IncidentStatus.WARNING.value), ) assert activity.incident == incident assert activity.type == IncidentActivityType.STATUS_CHANGE.value assert activity.user == self.user assert activity.value == six.text_type(IncidentStatus.CLOSED.value) assert activity.previous_value == six.text_type( IncidentStatus.WARNING.value) self.assert_notifications_sent(activity) assert not self.record_event.called def test_comment(self): incident = self.create_incident() comment = "hello" with self.assertChanges( lambda: IncidentSubscription.objects.filter( incident=incident, user=self.user).exists(), before=False, after=True, ): self.record_event.reset_mock() activity = create_incident_activity(incident, IncidentActivityType.COMMENT, user=self.user, comment=comment) assert activity.incident == incident assert activity.type == IncidentActivityType.COMMENT.value assert activity.user == self.user assert activity.comment == comment assert activity.value is None assert activity.previous_value is None self.assert_notifications_sent(activity) assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentCommentCreatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(incident.type), "user_id": six.text_type(self.user.id), "activity_id": six.text_type(activity.id), } def test_mentioned_user_ids(self): incident = self.create_incident() mentioned_member = self.create_user() subscribed_mentioned_member = self.create_user() IncidentSubscription.objects.create(incident=incident, user=subscribed_mentioned_member) comment = "hello **@%s** and **@%s**" % ( mentioned_member.username, subscribed_mentioned_member.username, ) with self.assertChanges( lambda: IncidentSubscription.objects.filter( incident=incident, user=mentioned_member).exists(), before=False, after=True, ): self.record_event.reset_mock() activity = create_incident_activity( incident, IncidentActivityType.COMMENT, user=self.user, comment=comment, mentioned_user_ids=[ mentioned_member.id, subscribed_mentioned_member.id ], ) assert activity.incident == incident assert activity.type == IncidentActivityType.COMMENT.value assert activity.user == self.user assert activity.comment == comment assert activity.value is None assert activity.previous_value is None self.assert_notifications_sent(activity) assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentCommentCreatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(incident.type), "user_id": six.text_type(self.user.id), "activity_id": six.text_type(activity.id), }
class CreateIncidentActivityTest(TestCase, BaseIncidentsTest): send_subscriber_notifications = patcher( 'sentry.incidents.logic.send_subscriber_notifications') def assert_notifications_sent(self, activity): self.send_subscriber_notifications.apply_async.assert_called_once_with( kwargs={'activity_id': activity.id}, countdown=10, ) def test_no_snapshot(self): incident = self.create_incident() activity = create_incident_activity( incident, IncidentActivityType.STATUS_CHANGE, user=self.user, value=six.text_type(IncidentStatus.CLOSED.value), previous_value=six.text_type(IncidentStatus.OPEN.value), ) assert activity.incident == incident assert activity.type == IncidentActivityType.STATUS_CHANGE.value assert activity.user == self.user assert activity.value == six.text_type(IncidentStatus.CLOSED.value) assert activity.previous_value == six.text_type( IncidentStatus.OPEN.value) self.assert_notifications_sent(activity) def test_snapshot(self): self.create_event(self.now - timedelta(minutes=2)) self.create_event(self.now - timedelta(minutes=2)) self.create_event(self.now - timedelta(minutes=1)) # Define events outside incident range. Should be included in the # snapshot self.create_event(self.now - timedelta(minutes=20)) self.create_event(self.now - timedelta(minutes=30)) # Too far out, should be excluded self.create_event(self.now - timedelta(minutes=100)) incident = self.create_incident(date_started=self.now - timedelta(minutes=5), query='', projects=[self.project]) event_stats_snapshot = create_initial_event_stats_snapshot(incident) activity = create_incident_activity( incident, IncidentActivityType.CREATED, event_stats_snapshot=event_stats_snapshot, ) assert activity.incident == incident assert activity.type == IncidentActivityType.CREATED.value assert activity.value is None assert activity.previous_value is None assert event_stats_snapshot == activity.event_stats_snapshot self.assert_notifications_sent(activity) def test_comment(self): incident = self.create_incident() comment = 'hello' with self.assertChanges( lambda: IncidentSubscription.objects.filter( incident=incident, user=self.user, ).exists(), before=False, after=True, ): activity = create_incident_activity( incident, IncidentActivityType.COMMENT, user=self.user, comment=comment, ) assert activity.incident == incident assert activity.type == IncidentActivityType.COMMENT.value assert activity.user == self.user assert activity.comment == comment assert activity.value is None assert activity.previous_value is None self.assert_notifications_sent(activity)
class ProcessUpdateTest(TestCase): metrics = patcher("sentry.incidents.subscription_processor.metrics") def setUp(self): super(ProcessUpdateTest, self).setUp() self.old_handlers = AlertRuleTriggerAction._type_registrations AlertRuleTriggerAction._type_registrations = {} self.email_action_handler = Mock() AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])( self.email_action_handler ) self._run_tasks = self.tasks() self._run_tasks.__enter__() def tearDown(self): super(ProcessUpdateTest, self).tearDown() AlertRuleTriggerAction._type_registrations = self.old_handlers self._run_tasks.__exit__(None, None, None) @fixture def other_project(self): return self.create_project() @fixture def sub(self): return self.rule.query_subscriptions.filter(project=self.project).get() @fixture def other_sub(self): return self.rule.query_subscriptions.filter(project=self.other_project).get() @fixture def rule(self): rule = create_alert_rule( self.organization, [self.project, self.other_project], "some rule", query="", aggregation=QueryAggregations.TOTAL, time_window=1, threshold_period=1, ) # Make sure the trigger exists trigger = create_alert_rule_trigger( rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10 ) create_alert_rule_trigger_action( trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, six.text_type(self.user.id), ) return rule @fixture def trigger(self): return self.rule.alertruletrigger_set.get() @fixture def action(self): return self.trigger.alertruletriggeraction_set.get() def build_subscription_update(self, subscription, time_delta=None, value=None): if time_delta is not None: timestamp = timezone.now() + time_delta else: timestamp = timezone.now() timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0) data = {} if subscription: aggregation_type = query_aggregation_to_snuba[ QueryAggregations(subscription.aggregation) ] value = randint(0, 100) if value is None else value data = {aggregation_type[2]: value} values = {"data": [data]} return { "subscription_id": subscription.subscription_id if subscription else uuid4().hex, "values": values, "timestamp": timestamp, "interval": 1, "partition": 1, "offset": 1, } def send_update(self, rule, value, time_delta=None, subscription=None): self.email_action_handler.reset_mock() if time_delta is None: time_delta = timedelta() if subscription is None: subscription = self.sub processor = SubscriptionProcessor(subscription) message = self.build_subscription_update(subscription, value=value, time_delta=time_delta) processor.process_update(message) return processor def assert_trigger_exists_with_status(self, incident, trigger, status): assert IncidentTrigger.objects.filter( incident=incident, alert_rule_trigger=trigger, status=status.value ).exists() def assert_trigger_does_not_exist_for_incident(self, incident, trigger): assert not IncidentTrigger.objects.filter( incident=incident, alert_rule_trigger=trigger ).exists() def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None): if incidents_to_exclude is None: incidents_to_exclude = [] assert ( not IncidentTrigger.objects.filter(alert_rule_trigger=trigger) .exclude(incident__in=incidents_to_exclude) .exists() ) def assert_action_handler_called_with_actions(self, incident, actions, project=None): project = self.project if project is None else project if not actions: if not incident: assert not self.email_action_handler.called else: for call_args in self.email_action_handler.call_args_list: assert call_args[0][1] != incident else: self.email_action_handler.assert_has_calls( [call(action, incident, project) for action in actions], any_order=True ) def assert_actions_fired_for_incident(self, incident, actions=None, project=None): actions = [] if actions is None else actions project = self.project if project is None else project self.assert_action_handler_called_with_actions(incident, actions, project) assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list) def assert_actions_resolved_for_incident(self, incident, actions=None, project=None): project = self.project if project is None else project actions = [] if actions is None else actions self.assert_action_handler_called_with_actions(incident, actions, project) assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list) def assert_no_active_incident(self, rule, subscription=None): assert not self.active_incident_exists(rule, subscription=subscription) def assert_active_incident(self, rule, subscription=None): incidents = self.active_incident_exists(rule, subscription=subscription) assert incidents return incidents[0] def active_incident_exists(self, rule, subscription=None): if subscription is None: subscription = self.sub return list( Incident.objects.filter( type=IncidentType.ALERT_TRIGGERED.value, alert_rule=rule, projects=subscription.project, ).exclude(status=IncidentStatus.CLOSED.value) ) def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0): assert processor.trigger_alert_counts[trigger.id] == alert_triggers assert processor.trigger_resolve_counts[trigger.id] == resolve_triggers alert_stats, resolve_stats = get_alert_rule_stats( processor.alert_rule, processor.subscription, [trigger] )[1:] assert alert_stats[trigger.id] == alert_triggers assert resolve_stats[trigger.id] == resolve_triggers def test_removed_alert_rule(self): message = self.build_subscription_update(self.sub) self.rule.delete() SubscriptionProcessor(self.sub).process_update(message) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.no_alert_rule_for_subscription" ) # TODO: Check subscription is deleted once we start doing that def test_skip_already_processed_update(self): self.send_update(self.rule, self.trigger.alert_threshold) self.metrics.incr.reset_mock() self.send_update(self.rule, self.trigger.alert_threshold) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update" ) self.metrics.incr.reset_mock() self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1)) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update" ) self.metrics.incr.reset_mock() self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1)) self.metrics.incr.assert_not_called() # NOQA def test_no_alert(self): rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(self.rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) def test_alert(self): # Verify that an alert rule that only expects a single update to be over the # alert threshold triggers correctly rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) def test_alert_multiple_threshold_periods(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold triggers correctly rule = self.rule trigger = self.trigger rule.update(threshold_period=2) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) def test_alert_multiple_triggers_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold doesn't trigger if there are two updates that are above with # an update that is below the threshold in the middle rule = self.rule rule.update(threshold_period=2) trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) def test_no_active_incident_resolve(self): # Test that we don't track stats for resolving if there are no active incidents # related to the alert rule. rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.resolve_threshold - 1) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) def test_resolve(self): # Verify that an alert rule that only expects a single update to be under the # resolve threshold triggers correctly rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_resolve_multiple_threshold_periods(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold triggers correctly rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) rule.update(threshold_period=2) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 1) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_resolve_multiple_threshold_periods_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold doesn't trigger if there's two updates that are below with # an update that is above the threshold in the middle rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) rule.update(threshold_period=2) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 0, 1) self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, trigger.resolve_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 1) self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) def test_reversed_threshold_alert(self): # Test that inverting thresholds correctly alerts rule = self.rule trigger = self.trigger trigger.update(threshold_type=AlertRuleThresholdType.BELOW.value) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) def test_reversed_threshold_resolve(self): # Test that inverting thresholds correctly resolves rule = self.rule trigger = self.trigger trigger.update(threshold_type=AlertRuleThresholdType.BELOW.value) processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, trigger.resolve_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_multiple_subscriptions_do_not_conflict(self): # Verify that multiple subscriptions associated with a rule don't conflict with # each other rule = self.rule rule.update(threshold_period=2) trigger = self.trigger # Send an update through for the first subscription. This shouldn't trigger an # incident, since we need two consecutive updates that are over the threshold. processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) # Have an update come through for the other sub. This shouldn't influence the original processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(self.trigger) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) # Send another update through for the first subscription. This should trigger an # incident for just this subscription. processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_does_not_exist(self.trigger, [incident]) # Send another update through for the second subscription. This should trigger an # incident for just this subscription. processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) other_incident = self.assert_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project) # Now we want to test that resolving is isolated. Send another update through # for the first subscription. processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) other_incident = self.assert_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(other_incident, []) processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) other_incident = self.assert_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(other_incident, []) # This second update for the second subscription should resolve its incident, # but not the incident from the first subscription. processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project) # This second update for the first subscription should resolve its incident now. processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED) self.assert_action_handler_called_with_actions(other_incident, []) def test_multiple_triggers(self): rule = self.rule rule.update(threshold_period=2) trigger = self.trigger other_trigger = create_alert_rule_trigger( self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50 ) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 1, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(trigger) self.assert_trigger_does_not_exist(other_trigger) self.assert_action_handler_called_with_actions(None, []) # This should cause both to increment, although only `trigger` should fire. processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 1, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_does_not_exist(other_trigger) self.assert_actions_fired_for_incident(incident, [self.action]) # Now only `other_trigger` should increment and fire. processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [other_action]) # Now send through two updates where we're below threshold for `other_trigger`. # The trigger should end up resolved, but the incident should still be active processor = self.send_update( rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update( rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [other_action]) # Now we push the other trigger below the resolve threshold twice. This should # close the incident. processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-5), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 1) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-4), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_multiple_triggers_at_same_time(self): # Check that both triggers fire if an update comes through that exceeds both of # their thresholds rule = self.rule trigger = self.trigger other_trigger = create_alert_rule_trigger( self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50 ) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action, other_action]) def test_multiple_triggers_one_with_no_resolve(self): # Check that both triggers fire if an update comes through that exceeds both of # their thresholds rule = self.rule trigger = self.trigger other_trigger = create_alert_rule_trigger( self.rule, "hello", AlertRuleThresholdType.ABOVE, 200 ) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_resolved_for_incident(incident, [self.action])
class ProcessUpdateTest(TestCase): metrics = patcher("sentry.incidents.subscription_processor.metrics") slack_client = patcher("sentry.integrations.slack.utils.SlackClient.post") def setUp(self): super().setUp() self.old_handlers = AlertRuleTriggerAction._type_registrations AlertRuleTriggerAction._type_registrations = {} self.email_action_handler = Mock() AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])( self.email_action_handler ) self._run_tasks = self.tasks() self._run_tasks.__enter__() def tearDown(self): super().tearDown() AlertRuleTriggerAction._type_registrations = self.old_handlers self._run_tasks.__exit__(None, None, None) @fixture def other_project(self): return self.create_project() @fixture def sub(self): return self.rule.snuba_query.subscriptions.filter(project=self.project).get() @fixture def other_sub(self): return self.rule.snuba_query.subscriptions.filter(project=self.other_project).get() @fixture def rule(self): rule = self.create_alert_rule( projects=[self.project, self.other_project], name="some rule", query="", aggregate="count()", time_window=1, threshold_type=AlertRuleThresholdType.ABOVE, resolve_threshold=10, threshold_period=1, ) # Make sure the trigger exists trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100) create_alert_rule_trigger_action( trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, str(self.user.id), ) return rule @fixture def trigger(self): return self.rule.alertruletrigger_set.get() @fixture def action(self): return self.trigger.alertruletriggeraction_set.get() def build_subscription_update(self, subscription, time_delta=None, value=EMPTY): if time_delta is not None: timestamp = timezone.now() + time_delta else: timestamp = timezone.now() timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0) data = {} if subscription: data = {"some_col_name": randint(0, 100) if value is EMPTY else value} values = {"data": [data]} return { "subscription_id": subscription.subscription_id if subscription else uuid4().hex, "values": values, "timestamp": timestamp, "interval": 1, "partition": 1, "offset": 1, } def send_update(self, rule, value, time_delta=None, subscription=None): self.email_action_handler.reset_mock() if time_delta is None: time_delta = timedelta() if subscription is None: subscription = self.sub processor = SubscriptionProcessor(subscription) message = self.build_subscription_update(subscription, value=value, time_delta=time_delta) with self.feature( ["organizations:incidents", "organizations:performance-view"] ), self.capture_on_commit_callbacks(execute=True): processor.process_update(message) return processor def assert_slack_calls(self, trigger_labels): expected = [f"{label}: some rule 2" for label in trigger_labels] actual = [ json.loads(call_kwargs["data"]["attachments"])[0]["title"] for (_, call_kwargs) in self.slack_client.call_args_list ] assert expected == actual self.slack_client.reset_mock() def assert_trigger_exists_with_status(self, incident, trigger, status): assert IncidentTrigger.objects.filter( incident=incident, alert_rule_trigger=trigger, status=status.value ).exists() def assert_trigger_does_not_exist_for_incident(self, incident, trigger): assert not IncidentTrigger.objects.filter( incident=incident, alert_rule_trigger=trigger ).exists() def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None): if incidents_to_exclude is None: incidents_to_exclude = [] assert ( not IncidentTrigger.objects.filter(alert_rule_trigger=trigger) .exclude(incident__in=incidents_to_exclude) .exists() ) def assert_action_handler_called_with_actions(self, incident, actions, project=None): project = self.project if project is None else project if not actions: if not incident: assert not self.email_action_handler.called else: for call_args in self.email_action_handler.call_args_list: assert call_args[0][1] != incident else: self.email_action_handler.assert_has_calls( [call(action, incident, project) for action in actions], any_order=True ) def assert_actions_fired_for_incident(self, incident, actions=None, project=None): actions = [] if actions is None else actions project = self.project if project is None else project self.assert_action_handler_called_with_actions(incident, actions, project) assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list) def assert_actions_resolved_for_incident(self, incident, actions=None, project=None): project = self.project if project is None else project actions = [] if actions is None else actions self.assert_action_handler_called_with_actions(incident, actions, project) assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list) def assert_no_active_incident(self, rule, subscription=None): assert not self.active_incident_exists(rule, subscription=subscription) def assert_active_incident(self, rule, subscription=None): incidents = self.active_incident_exists(rule, subscription=subscription) assert incidents return incidents[0] def active_incident_exists(self, rule, subscription=None): if subscription is None: subscription = self.sub return list( Incident.objects.filter( type=IncidentType.ALERT_TRIGGERED.value, alert_rule=rule, projects=subscription.project, ).exclude(status=IncidentStatus.CLOSED.value) ) def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0): assert processor.trigger_alert_counts[trigger.id] == alert_triggers alert_stats, resolve_stats = get_alert_rule_stats( processor.alert_rule, processor.subscription, [trigger] )[1:] assert alert_stats[trigger.id] == alert_triggers assert resolve_stats[trigger.id] == resolve_triggers def test_removed_alert_rule(self): message = self.build_subscription_update(self.sub) self.rule.delete() with self.feature(["organizations:incidents", "organizations:performance-view"]): SubscriptionProcessor(self.sub).process_update(message) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.no_alert_rule_for_subscription" ) # TODO: Check subscription is deleted once we start doing that def test_removed_project(self): message = self.build_subscription_update(self.sub) self.project.delete() with self.feature(["organizations:incidents", "organizations:performance-view"]): SubscriptionProcessor(self.sub).process_update(message) self.metrics.incr.assert_called_once_with("incidents.alert_rules.ignore_deleted_project") def test_no_feature(self): message = self.build_subscription_update(self.sub) SubscriptionProcessor(self.sub).process_update(message) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.ignore_update_missing_incidents" ) def test_no_feature_performance(self): self.sub.snuba_query.dataset = "transactions" message = self.build_subscription_update(self.sub) with self.feature("organizations:incidents"): SubscriptionProcessor(self.sub).process_update(message) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.ignore_update_missing_incidents_performance" ) def test_skip_already_processed_update(self): self.send_update(self.rule, self.trigger.alert_threshold) self.metrics.incr.reset_mock() self.send_update(self.rule, self.trigger.alert_threshold) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update" ) self.metrics.incr.reset_mock() self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1)) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update" ) self.metrics.incr.reset_mock() self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1)) self.metrics.incr.assert_not_called() # NOQA def test_no_alert(self): rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(self.rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) def test_alert(self): # Verify that an alert rule that only expects a single update to be over the # alert threshold triggers correctly rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) assert incident.date_started == ( timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window) ) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) def test_alert_dedupe(self): # Verify that an alert rule that only expects a single update to be over the # alert threshold triggers correctly rule = self.rule c_trigger = self.trigger c_action_2 = create_alert_rule_trigger_action( self.trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, str(self.user.id), ) w_trigger = create_alert_rule_trigger( self.rule, WARNING_TRIGGER_LABEL, c_trigger.alert_threshold - 10 ) create_alert_rule_trigger_action( w_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, str(self.user.id), ) processor = self.send_update(rule, c_trigger.alert_threshold + 1) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) assert incident.date_started == ( timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window) ) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [c_action_2]) def test_alert_nullable(self): # Verify that an alert rule that only expects a single update to be over the # alert threshold triggers correctly rule = self.rule self.trigger processor = self.send_update(rule, None) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) def test_alert_multiple_threshold_periods(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold triggers correctly rule = self.rule trigger = self.trigger rule.update(threshold_period=2) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) def test_alert_multiple_triggers_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold doesn't trigger if there are two updates that are above with # an update that is below the threshold in the middle rule = self.rule rule.update(threshold_period=2) trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) def test_no_active_incident_resolve(self): # Test that we don't track stats for resolving if there are no active incidents # related to the alert rule. rule = self.rule trigger = self.trigger processor = self.send_update(rule, rule.resolve_threshold - 1) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(trigger) self.assert_action_handler_called_with_actions(None, []) def test_resolve(self): # Verify that an alert rule that only expects a single update to be under the # resolve threshold triggers correctly rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_resolve_multiple_threshold_periods(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold triggers correctly rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) rule.update(threshold_period=2) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 1) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_resolve_multiple_threshold_periods_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold doesn't trigger if there's two updates that are below with # an update that is above the threshold in the middle rule = self.rule trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) rule.update(threshold_period=2) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 0, 1) self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 1) self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) def test_auto_resolve(self): # Verify that we resolve an alert rule automatically even if no resolve # threshold is set rule = self.rule rule.update(resolve_threshold=None) trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_auto_resolve_percent_boundary(self): # Verify that we resolve an alert rule automatically even if no resolve # threshold is set rule = self.rule rule.update(resolve_threshold=None) trigger = self.trigger trigger.update(alert_threshold=0.5) processor = self.send_update(rule, trigger.alert_threshold + 0.1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1)) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_auto_resolve_boundary(self): # Verify that we resolve an alert rule automatically if the value hits the # original alert trigger value rule = self.rule rule.update(resolve_threshold=None) trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1)) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_auto_resolve_reversed(self): # Test auto resolving works correctly when threshold is reversed rule = self.rule rule.update(resolve_threshold=None, threshold_type=AlertRuleThresholdType.BELOW.value) trigger = self.trigger processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_auto_resolve_multiple_trigger(self): # Test auto resolving works correctly when multiple triggers are present. rule = self.rule rule.update(resolve_threshold=None) trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", trigger.alert_threshold - 10) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update(rule, other_trigger.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action, other_action]) def test_reversed_threshold_alert(self): # Test that inverting thresholds correctly alerts rule = self.rule trigger = self.trigger rule.update(threshold_type=AlertRuleThresholdType.BELOW.value) trigger.update(alert_threshold=rule.resolve_threshold + 1) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_does_not_exist(trigger) self.assert_action_handler_called_with_actions(None, []) processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) def test_reversed_threshold_resolve(self): # Test that inverting thresholds correctly resolves rule = self.rule trigger = self.trigger rule.update(threshold_type=AlertRuleThresholdType.BELOW.value) trigger.update(alert_threshold=rule.resolve_threshold + 1) processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) def test_multiple_subscriptions_do_not_conflict(self): # Verify that multiple subscriptions associated with a rule don't conflict with # each other rule = self.rule rule.update(threshold_period=2) trigger = self.trigger # Send an update through for the first subscription. This shouldn't trigger an # incident, since we need two consecutive updates that are over the threshold. processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) # Have an update come through for the other sub. This shouldn't influence the original processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 1, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(self.trigger) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_does_not_exist(self.trigger) self.assert_action_handler_called_with_actions(None, []) # Send another update through for the first subscription. This should trigger an # incident for just this subscription. processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action]) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_does_not_exist(self.trigger, [incident]) # Send another update through for the second subscription. This should trigger an # incident for just this subscription. processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) other_incident = self.assert_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project) # Now we want to test that resolving is isolated. Send another update through # for the first subscription. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) other_incident = self.assert_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(other_incident, []) processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) other_incident = self.assert_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(other_incident, []) # This second update for the second subscription should resolve its incident, # but not the incident from the first subscription. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project) # This second update for the first subscription should resolve its incident now. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub ) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action]) self.assert_no_active_incident(rule, self.other_sub) self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED) self.assert_action_handler_called_with_actions(other_incident, []) def test_multiple_triggers(self): rule = self.rule rule.update(threshold_period=2) trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", 200) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 1, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(trigger) self.assert_trigger_does_not_exist(other_trigger) self.assert_action_handler_called_with_actions(None, []) # This should cause both to increment, although only `trigger` should fire. processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 1, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_does_not_exist(other_trigger) self.assert_actions_fired_for_incident(incident, [self.action]) # Now only `other_trigger` should increment and fire. processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [other_action]) # Now send through two updates where we're below threshold for the rule. This # should resolve all triggers and the incident should be closed. processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 1) self.assert_trigger_counts(processor, other_trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action, other_action]) def test_slack_multiple_triggers_critical_before_warning(self): """ Test that ensures that when we get a critical update is sent followed by a warning update, the warning update is not swallowed and an alert is triggered as a warning alert granted the count is above the warning trigger threshold """ from sentry.incidents.action_handlers import SlackActionHandler slack_handler = SlackActionHandler # Create Slack Integration integration = Integration.objects.create( provider="slack", name="Team A", external_id="TXXXXXXX1", metadata={ "access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx", "installation_type": "born_as_bot", }, ) integration.add_organization(self.project.organization, self.user) # Register Slack Handler AlertRuleTriggerAction.register_type( "slack", AlertRuleTriggerAction.Type.SLACK, [AlertRuleTriggerAction.TargetType.SPECIFIC], integration_provider="slack", )(slack_handler) rule = self.create_alert_rule( projects=[self.project, self.other_project], name="some rule 2", query="", aggregate="count()", time_window=1, threshold_type=AlertRuleThresholdType.ABOVE, resolve_threshold=10, threshold_period=1, ) trigger = create_alert_rule_trigger(rule, "critical", 100) trigger_warning = create_alert_rule_trigger(rule, "warning", 10) for t in [trigger, trigger_warning]: create_alert_rule_trigger_action( t, AlertRuleTriggerAction.Type.SLACK, AlertRuleTriggerAction.TargetType.SPECIFIC, integration=integration, input_channel_id="#workflow", ) # Send Critical Update self.send_update( rule, trigger.alert_threshold + 5, timedelta(minutes=-10), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_slack_calls(["Critical"]) # Send Warning Update self.send_update( rule, trigger_warning.alert_threshold + 5, timedelta(minutes=0), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_slack_calls(["Warning"]) self.assert_active_incident(rule) def test_slack_multiple_triggers_critical_fired_twice_before_warning(self): """ Test that ensures that when we get a critical update is sent followed by a warning update, the warning update is not swallowed and an alert is triggered as a warning alert granted the count is above the warning trigger threshold """ from sentry.incidents.action_handlers import SlackActionHandler slack_handler = SlackActionHandler # Create Slack Integration integration = Integration.objects.create( provider="slack", name="Team A", external_id="TXXXXXXX1", metadata={ "access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx", "installation_type": "born_as_bot", }, ) integration.add_organization(self.project.organization, self.user) # Register Slack Handler AlertRuleTriggerAction.register_type( "slack", AlertRuleTriggerAction.Type.SLACK, [AlertRuleTriggerAction.TargetType.SPECIFIC], integration_provider="slack", )(slack_handler) rule = self.create_alert_rule( projects=[self.project, self.other_project], name="some rule 2", query="", aggregate="count()", time_window=1, threshold_type=AlertRuleThresholdType.ABOVE, resolve_threshold=10, threshold_period=1, ) trigger = create_alert_rule_trigger(rule, "critical", 100) trigger_warning = create_alert_rule_trigger(rule, "warning", 10) for t in [trigger, trigger_warning]: create_alert_rule_trigger_action( t, AlertRuleTriggerAction.Type.SLACK, AlertRuleTriggerAction.TargetType.SPECIFIC, integration=integration, input_channel_id="#workflow", ) self.assert_slack_calls([]) # Send update above critical self.send_update( rule, trigger.alert_threshold + 5, timedelta(minutes=-10), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_slack_calls(["Critical"]) # Send second update above critical self.send_update( rule, trigger.alert_threshold + 6, timedelta(minutes=-9), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_slack_calls([]) # Send update below critical but above warning self.send_update( rule, trigger_warning.alert_threshold + 5, timedelta(minutes=0), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_active_incident(rule) self.assert_slack_calls(["Warning"]) def test_multiple_triggers_at_same_time(self): # Check that both triggers fire if an update comes through that exceeds both of # their thresholds rule = self.rule trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", 200) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action, other_action]) def test_multiple_triggers_resolve_separately(self): # Check that resolve triggers fire separately rule = self.rule trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", 200) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update( rule, other_trigger.alert_threshold - 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [other_action]) processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-8), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action])
class ProcessUpdateTest(TestCase): metrics = patcher("sentry.incidents.subscription_processor.metrics") @fixture def subscription(self): subscription = QuerySubscription.objects.create( project=self.project, type=INCIDENTS_SNUBA_SUBSCRIPTION_TYPE, subscription_id="some_id", dataset=SnubaDatasets.EVENTS.value, query="", aggregations=[AlertRuleAggregations.TOTAL.value], time_window=1, resolution=1, ) return subscription @fixture def rule(self): rule = create_alert_rule( self.project, "some rule", AlertRuleThresholdType.ABOVE, query="", aggregations=[AlertRuleAggregations.TOTAL], time_window=1, alert_threshold=100, resolve_threshold=10, threshold_period=1, ) rule.update(query_subscription=self.subscription) return rule def build_subscription_update(self, subscription=None, time_delta=None, value=None): if time_delta is not None: timestamp = int(to_timestamp(timezone.now() + time_delta)) else: timestamp = int(time()) values = {} if subscription: aggregation_type = alert_aggregation_to_snuba[ AlertRuleAggregations(subscription.aggregations[0])] value = randint(0, 100) if value is None else value values = {aggregation_type[2]: value} return { "subscription_id": subscription.subscription_id if subscription else uuid4().hex, "values": values, "timestamp": timestamp, "interval": 1, "partition": 1, "offset": 1, } def send_update(self, rule, value, time_delta=None): if time_delta is None: time_delta = timedelta() subscription = rule.query_subscription processor = SubscriptionProcessor(subscription) message = self.build_subscription_update(subscription, value=value, time_delta=time_delta) processor.process_update(message) return processor def assert_no_active_incident(self, rule): assert not self.active_incident_exists(rule) def assert_active_incident(self, rule): assert self.active_incident_exists(rule) def active_incident_exists(self, rule): return Incident.objects.filter( type=IncidentType.ALERT_TRIGGERED.value, status=IncidentStatus.OPEN.value, alert_rule=rule, ).exists() def assert_trigger_counts(self, processor, alert_triggers=0, resolve_triggers=0): assert processor.alert_triggers == alert_triggers assert processor.resolve_triggers == resolve_triggers assert get_alert_rule_stats( processor.alert_rule)[1:] == (alert_triggers, resolve_triggers) def test_removed_alert_rule(self): message = self.build_subscription_update() SubscriptionProcessor(self.subscription).process_update(message) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.no_alert_rule_for_subscription") # TODO: Check subscription is deleted once we start doing that def test_skip_already_processed_update(self): self.send_update(self.rule, self.rule.alert_threshold) self.metrics.incr.reset_mock() self.send_update(self.rule, self.rule.alert_threshold) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update") self.metrics.incr.reset_mock() self.send_update(self.rule, self.rule.alert_threshold, timedelta(hours=-1)) self.metrics.incr.assert_called_once_with( "incidents.alert_rules.skipping_already_processed_update") self.metrics.incr.reset_mock() self.send_update(self.rule, self.rule.alert_threshold, timedelta(hours=1)) self.metrics.incr.assert_not_called() # NOQA def test_no_alert(self): rule = self.rule processor = self.send_update(rule, rule.alert_threshold) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(self.rule) def test_alert(self): # Verify that an alert rule that only expects a single update to be over the # alert threshold triggers correctly rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) def test_alert_multiple_triggers(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold triggers correctly rule = self.rule rule.update(threshold_period=2) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) def test_alert_multiple_triggers_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be over the # alert threshold doesn't trigger if there are two updates that are above with # an update that is below the threshold in the middle rule = self.rule rule.update(threshold_period=2) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 1, 0) self.assert_no_active_incident(rule) def test_no_active_incident_resolve(self): # Test that we don't track stats for resolving if there are no active incidents # related to the alert rule. rule = self.rule processor = self.send_update(rule, rule.resolve_threshold - 1) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_resolve(self): # Verify that an alert rule that only expects a single update to be under the # resolve threshold triggers correctly rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_resolve_multiple_triggers(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold triggers correctly rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) rule.update(threshold_period=2) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) def test_resolve_multiple_triggers_non_consecutive(self): # Verify that a rule that expects two consecutive updates to be under the # resolve threshold doesn't trigger if there's two updates that are below with # an update that is above the threshold in the middle rule = self.rule processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-4)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) rule.update(threshold_period=2) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 1) self.assert_active_incident(rule) def test_reversed_threshold_alert(self): # Test that inverting thresholds correctly alerts rule = self.rule rule.update(threshold_type=AlertRuleThresholdType.BELOW.value) processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule) processor = self.send_update(rule, rule.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) def test_reversed_threshold_resolve(self): # Test that inverting thresholds correctly resolves rule = self.rule rule.update(threshold_type=AlertRuleThresholdType.BELOW.value) processor = self.send_update(rule, rule.alert_threshold - 1, timedelta(minutes=-3)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, 0, 0) self.assert_active_incident(rule) processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, 0, 0) self.assert_no_active_incident(rule)
class UpdateIncidentStatus(TestCase): record_event = patcher('sentry.analytics.base.Analytics.record_event') def get_most_recent_incident_activity(self, incident): return IncidentActivity.objects.filter( incident=incident).order_by('-id')[:1].get() def test_status_already_set(self): incident = self.create_incident(status=IncidentStatus.OPEN.value) with self.assertRaises(StatusAlreadyChangedError): update_incident_status(incident, IncidentStatus.OPEN) def run_test( self, incident, status, expected_date_closed, user=None, comment=None, ): prev_status = incident.status self.record_event.reset_mock() update_incident_status(incident, status, user=user, comment=comment) incident = Incident.objects.get(id=incident.id) assert incident.status == status.value assert incident.date_closed == expected_date_closed activity = self.get_most_recent_incident_activity(incident) assert activity.type == IncidentActivityType.STATUS_CHANGE.value assert activity.user == user if user: assert IncidentSubscription.objects.filter(incident=incident, user=user).exists() assert activity.value == six.text_type(status.value) assert activity.previous_value == six.text_type(prev_status) assert activity.comment == comment assert activity.event_stats_snapshot is None assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentStatusUpdatedEvent) assert event.data == { 'organization_id': six.text_type(self.organization.id), 'incident_id': six.text_type(incident.id), 'incident_type': six.text_type(incident.type), 'prev_status': six.text_type(prev_status), 'status': six.text_type(incident.status), } def test_closed(self): incident = self.create_incident() self.run_test(incident, IncidentStatus.CLOSED, timezone.now()) def test_reopened(self): incident = self.create_incident(status=IncidentStatus.CLOSED.value, date_closed=timezone.now()) self.run_test(incident, IncidentStatus.OPEN, None) def test_all_params(self): incident = self.create_incident() self.run_test( incident, IncidentStatus.CLOSED, timezone.now(), user=self.user, comment='lol', )
class CreateIncidentActivityTest(TestCase, BaseIncidentsTest): send_subscriber_notifications = patcher("sentry.incidents.tasks.send_subscriber_notifications") record_event = patcher("sentry.analytics.base.Analytics.record_event") def assert_notifications_sent(self, activity): self.send_subscriber_notifications.apply_async.assert_called_once_with( kwargs={"activity_id": activity.id}, countdown=10 ) def test_no_snapshot(self): incident = self.create_incident() self.record_event.reset_mock() activity = create_incident_activity( incident, IncidentActivityType.STATUS_CHANGE, user=self.user, value=six.text_type(IncidentStatus.CLOSED.value), previous_value=six.text_type(IncidentStatus.OPEN.value), ) assert activity.incident == incident assert activity.type == IncidentActivityType.STATUS_CHANGE.value assert activity.user == self.user assert activity.value == six.text_type(IncidentStatus.CLOSED.value) assert activity.previous_value == six.text_type(IncidentStatus.OPEN.value) self.assert_notifications_sent(activity) assert not self.record_event.called def test_snapshot(self): self.create_event(self.now - timedelta(minutes=2)) self.create_event(self.now - timedelta(minutes=2)) self.create_event(self.now - timedelta(minutes=1)) # Define events outside incident range. Should be included in the # snapshot self.create_event(self.now - timedelta(minutes=20)) self.create_event(self.now - timedelta(minutes=30)) # Too far out, should be excluded self.create_event(self.now - timedelta(minutes=100)) incident = self.create_incident( date_started=self.now - timedelta(minutes=5), query="", projects=[self.project] ) event_stats_snapshot = create_initial_event_stats_snapshot(incident) self.record_event.reset_mock() activity = create_incident_activity( incident, IncidentActivityType.CREATED, event_stats_snapshot=event_stats_snapshot ) assert activity.incident == incident assert activity.type == IncidentActivityType.CREATED.value assert activity.value is None assert activity.previous_value is None assert event_stats_snapshot == activity.event_stats_snapshot self.assert_notifications_sent(activity) assert not self.record_event.called def test_comment(self): incident = self.create_incident() comment = "hello" with self.assertChanges( lambda: IncidentSubscription.objects.filter(incident=incident, user=self.user).exists(), before=False, after=True, ): self.record_event.reset_mock() activity = create_incident_activity( incident, IncidentActivityType.COMMENT, user=self.user, comment=comment ) assert activity.incident == incident assert activity.type == IncidentActivityType.COMMENT.value assert activity.user == self.user assert activity.comment == comment assert activity.value is None assert activity.previous_value is None self.assert_notifications_sent(activity) assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentCommentCreatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(incident.type), "user_id": six.text_type(self.user.id), "activity_id": six.text_type(activity.id), } def test_mentioned_user_ids(self): incident = self.create_incident() mentioned_member = self.create_user() subscribed_mentioned_member = self.create_user() IncidentSubscription.objects.create(incident=incident, user=subscribed_mentioned_member) comment = "hello **@%s** and **@%s**" % ( mentioned_member.username, subscribed_mentioned_member.username, ) with self.assertChanges( lambda: IncidentSubscription.objects.filter( incident=incident, user=mentioned_member ).exists(), before=False, after=True, ): self.record_event.reset_mock() activity = create_incident_activity( incident, IncidentActivityType.COMMENT, user=self.user, comment=comment, mentioned_user_ids=[mentioned_member.id, subscribed_mentioned_member.id], ) assert activity.incident == incident assert activity.type == IncidentActivityType.COMMENT.value assert activity.user == self.user assert activity.comment == comment assert activity.value is None assert activity.previous_value is None self.assert_notifications_sent(activity) assert len(self.record_event.call_args_list) == 1 event = self.record_event.call_args[0][0] assert isinstance(event, IncidentCommentCreatedEvent) assert event.data == { "organization_id": six.text_type(self.organization.id), "incident_id": six.text_type(incident.id), "incident_type": six.text_type(incident.type), "user_id": six.text_type(self.user.id), "activity_id": six.text_type(activity.id), }