def test(self): label = "hello" threshold_type = AlertRuleThresholdType.ABOVE alert_threshold = 1000 resolve_threshold = 400 trigger = create_alert_rule_trigger(self.alert_rule, label, threshold_type, alert_threshold, resolve_threshold) assert trigger.label == label assert trigger.threshold_type == threshold_type.value assert trigger.alert_threshold == alert_threshold assert trigger.resolve_threshold == resolve_threshold assert not AlertRuleTriggerExclusion.objects.filter( alert_rule_trigger=trigger).exists()
def rule(self): with self.tasks(): rule = self.create_alert_rule( name="some rule", query="", aggregate="count()", time_window=1, threshold_period=1 ) trigger = create_alert_rule_trigger( rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10 ) create_alert_rule_trigger_action( trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, six.text_type(self.user.id), ) return rule
def test_excluded_projects(self): excluded_project = self.create_project(fire_project_created=True) alert_rule = self.create_alert_rule( projects=[self.project, excluded_project]) trigger = create_alert_rule_trigger( alert_rule, "hi", AlertRuleThresholdType.ABOVE, 100, excluded_projects=[excluded_project], ) # We should have only one exclusion exclusion = AlertRuleTriggerExclusion.objects.get( alert_rule_trigger=trigger) assert exclusion.query_subscription.project == excluded_project
def update(self, instance, validated_data): with transaction.atomic(): validated_data = self._remove_unchanged_fields( instance, validated_data) triggers_data = validated_data.pop("triggers") alert_rule = update_alert_rule(instance, **validated_data) # Delete triggers we don't have present in the updated data. trigger_ids = [x["id"] for x in triggers_data] AlertRuleTrigger.objects.filter(alert_rule=alert_rule).exclude( id__in=trigger_ids).delete() for trigger_data in triggers_data: actions_data = trigger_data.pop("actions") try: if "id" in trigger_data: trigger_instance = AlertRuleTrigger.objects.get( alert_rule=alert_rule, id=trigger_data["id"]) trigger_data.pop("id") trigger = update_alert_rule_trigger( trigger_instance, **trigger_data) else: trigger = create_alert_rule_trigger( alert_rule=alert_rule, **trigger_data) except AlertRuleTriggerLabelAlreadyUsedError: raise serializers.ValidationError( "This trigger label is already in use for this alert rule" ) # Delete actions we don't have present in the updated data. action_ids = [x["id"] for x in actions_data] AlertRuleTriggerAction.objects.filter( alert_rule_trigger=trigger).exclude( id__in=action_ids).delete() for action_data in actions_data: if "id" in action_data: action_instance = AlertRuleTriggerAction.objects.get( alert_rule_trigger=trigger, id=action_data["id"]) action_data.pop("id") update_alert_rule_trigger_action( action_instance, **action_data) else: create_alert_rule_trigger_action(trigger=trigger, **action_data) return alert_rule
def rule(self): with self.tasks(): rule = self.create_alert_rule( name="some rule", query="", aggregate="count()", time_window=1, threshold_period=1, resolve_threshold=10, ) trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100) create_alert_rule_trigger_action( trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, str(self.user.id), ) return rule
def rule(self): rule = create_alert_rule( self.organization, [self.project], "some rule", query="", aggregation=QueryAggregations.TOTAL, time_window=1, threshold_period=1, ) trigger = create_alert_rule_trigger( rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10 ) create_alert_rule_trigger_action( trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, six.text_type(self.user.id), ) return rule
def rule(self): rule = self.create_alert_rule( projects=[self.project, self.other_project], name="some rule", query="", aggregate="count()", time_window=1, threshold_type=AlertRuleThresholdType.ABOVE, resolve_threshold=10, threshold_period=1, ) # Make sure the trigger exists trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100) create_alert_rule_trigger_action( trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER, str(self.user.id), ) return rule
def test(self): alert_rule = self.create_alert_rule() trigger = create_alert_rule_trigger( alert_rule, "hi", AlertRuleThresholdType.ABOVE, 1000, 400, excluded_projects=[self.project], ) trigger_id = trigger.id assert AlertRuleTriggerExclusion.objects.filter( alert_rule_trigger=trigger, query_subscription__project=self.project).exists() delete_alert_rule_trigger(trigger) assert not AlertRuleTrigger.objects.filter(id=trigger_id).exists() assert not AlertRuleTriggerExclusion.objects.filter( alert_rule_trigger=trigger, query_subscription__project=self.project).exists()
def test(self): trigger = create_alert_rule_trigger(self.alert_rule, "hello", AlertRuleThresholdType.ABOVE, 1000, 400) label = "uh oh" threshold_type = AlertRuleThresholdType.BELOW alert_threshold = 2000 resolve_threshold = 800 trigger = update_alert_rule_trigger( trigger, label=label, threshold_type=threshold_type, alert_threshold=alert_threshold, resolve_threshold=resolve_threshold, ) assert trigger.label == label assert trigger.threshold_type == threshold_type.value assert trigger.alert_threshold == alert_threshold assert trigger.resolve_threshold == resolve_threshold
def test_complex_exclude_projects(self): excluded_project = self.create_project() other_project = self.create_project(fire_project_created=True) alert_rule = self.create_alert_rule( projects=[excluded_project, self.project, other_project] ) trigger = create_alert_rule_trigger( alert_rule, "hi", AlertRuleThresholdType.ABOVE, 1000, 400, excluded_projects=[excluded_project, self.project], ) update_alert_rule_trigger(trigger, excluded_projects=[other_project, excluded_project]) excluded_projects = [ exclusion.query_subscription.project for exclusion in trigger.exclusions.all() ] assert set(excluded_projects) == set([other_project, excluded_project])
def test_multiple_triggers_at_same_time(self): # Check that both triggers fire if an update comes through that exceeds both of # their thresholds rule = self.rule trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER) processor = self.send_update(rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_auto_resolve_multiple_trigger(self): # Test auto resolving works correctly when multiple triggers are present. rule = self.rule rule.update(resolve_threshold=None) trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", trigger.alert_threshold - 10) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2)) self.assert_trigger_counts(processor, self.trigger, 0, 0) incident = self.assert_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update(rule, other_trigger.alert_threshold - 1, timedelta(minutes=-1)) self.assert_trigger_counts(processor, self.trigger, 0, 0) self.assert_no_active_incident(rule) self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_resolve_separately(self): # Check that resolve triggers fire separately rule = self.rule trigger = self.trigger other_trigger = create_alert_rule_trigger(self.rule, "hello", 200) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [self.action, other_action]) processor = self.send_update( rule, other_trigger.alert_threshold - 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [other_action]) processor = self.send_update( rule, rule.resolve_threshold - 1, timedelta(minutes=-8), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action])
def create(self, validated_data): with transaction.atomic(): try: # TODO: Remove this, just temporary while we're supporting both fields. if "aggregation" not in validated_data: raise serializers.ValidationError( "aggregation is required") triggers_data = validated_data.pop("triggers") # TODO: User super.create and don't duplicate the aggreagation + duplicate name check? alert_rule = create_alert_rule( organization=self.context["organization"], **validated_data) for trigger_data in triggers_data: trigger_actions_data = trigger_data.pop("actions") trigger = create_alert_rule_trigger(alert_rule=alert_rule, **trigger_data) for actions_data in trigger_actions_data: create_alert_rule_trigger_action(trigger=trigger, **actions_data) return alert_rule except AlertRuleNameAlreadyUsedError: raise serializers.ValidationError( "This name is already in use for this project")
def create_alert_rule_trigger(alert_rule, label=None, alert_threshold=100): if not label: label = petname.Generate(2, " ", letters=10).title() return create_alert_rule_trigger(alert_rule, label, alert_threshold)
def trigger(self): return create_alert_rule_trigger(self.alert_rule, "", 100)
def test_existing_label(self): name = "uh oh" create_alert_rule_trigger(self.alert_rule, name, AlertRuleThresholdType.ABOVE, 100) with self.assertRaises(AlertRuleTriggerLabelAlreadyUsedError): create_alert_rule_trigger(self.alert_rule, name, AlertRuleThresholdType.ABOVE, 100)
def trigger(self): return create_alert_rule_trigger( self.alert_rule, "hello", AlertRuleThresholdType.ABOVE, 1000, 400 )
def test(self): alert_rule = self.create_alert_rule() trigger = create_alert_rule_trigger(alert_rule, "hi", AlertRuleThresholdType.ABOVE, 1000, 400) assert get_triggers_for_alert_rule(alert_rule).get() == trigger
def test_simple(self): alert_rule = self.create_alert_rule(resolve_threshold=200) trigger = create_alert_rule_trigger(alert_rule, "hi", 1000) result = serialize(trigger, serializer=DetailedAlertRuleTriggerSerializer()) self.assert_alert_rule_trigger_serialized(trigger, result) assert result["excludedProjects"] == []
def test_comparison_above(self): alert_rule = self.create_alert_rule(comparison_delta=60) trigger = create_alert_rule_trigger(alert_rule, "hi", 180) result = serialize(trigger) self.assert_alert_rule_trigger_serialized(trigger, result, 80)
def test_multiple_triggers(self): rule = self.rule rule.update(threshold_period=2) trigger = self.trigger other_trigger = create_alert_rule_trigger( self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50 ) other_action = create_alert_rule_trigger_action( other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER ) processor = self.send_update( rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 1, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_does_not_exist(trigger) self.assert_trigger_does_not_exist(other_trigger) self.assert_action_handler_called_with_actions(None, []) # This should cause both to increment, although only `trigger` should fire. processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 1, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_does_not_exist(other_trigger) self.assert_actions_fired_for_incident(incident, [self.action]) # Now only `other_trigger` should increment and fire. processor = self.send_update( rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_actions_fired_for_incident(incident, [other_action]) # Now send through two updates where we're below threshold for `other_trigger`. # The trigger should end up resolved, but the incident should still be active processor = self.send_update( rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 1) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update( rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [other_action]) # Now we push the other trigger below the resolve threshold twice. This should # close the incident. processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-5), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 1) self.assert_trigger_counts(processor, other_trigger, 0, 0) incident = self.assert_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_action_handler_called_with_actions(incident, []) processor = self.send_update( rule, trigger.resolve_threshold - 1, timedelta(minutes=-4), subscription=self.sub ) self.assert_trigger_counts(processor, trigger, 0, 0) self.assert_trigger_counts(processor, other_trigger, 0, 0) self.assert_no_active_incident(rule, self.sub) self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED) self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED) self.assert_actions_resolved_for_incident(incident, [self.action])
def test_decimal(self): alert_rule = self.create_alert_rule(resolve_threshold=200.70) trigger = create_alert_rule_trigger(alert_rule, "hi", 1000.50) result = serialize(trigger) self.assert_alert_rule_trigger_serialized(trigger, result)
def trigger(self): return create_alert_rule_trigger(self.alert_rule, CRITICAL_TRIGGER_LABEL, 100)
def test_slack_multiple_triggers_critical_fired_twice_before_warning(self): """ Test that ensures that when we get a critical update is sent followed by a warning update, the warning update is not swallowed and an alert is triggered as a warning alert granted the count is above the warning trigger threshold """ from sentry.incidents.action_handlers import SlackActionHandler slack_handler = SlackActionHandler # Create Slack Integration integration = Integration.objects.create( provider="slack", name="Team A", external_id="TXXXXXXX1", metadata={ "access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx", "installation_type": "born_as_bot", }, ) integration.add_organization(self.project.organization, self.user) # Register Slack Handler AlertRuleTriggerAction.register_type( "slack", AlertRuleTriggerAction.Type.SLACK, [AlertRuleTriggerAction.TargetType.SPECIFIC], integration_provider="slack", )(slack_handler) rule = self.create_alert_rule( projects=[self.project, self.other_project], name="some rule 2", query="", aggregate="count()", time_window=1, threshold_type=AlertRuleThresholdType.ABOVE, resolve_threshold=10, threshold_period=1, ) trigger = create_alert_rule_trigger(rule, "critical", 100) trigger_warning = create_alert_rule_trigger(rule, "warning", 10) for t in [trigger, trigger_warning]: create_alert_rule_trigger_action( t, AlertRuleTriggerAction.Type.SLACK, AlertRuleTriggerAction.TargetType.SPECIFIC, integration=integration, input_channel_id="#workflow", ) self.assert_slack_calls([]) # Send update above critical self.send_update( rule, trigger.alert_threshold + 5, timedelta(minutes=-10), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_slack_calls(["Critical"]) # Send second update above critical self.send_update( rule, trigger.alert_threshold + 6, timedelta(minutes=-9), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_slack_calls([]) # Send update below critical but above warning self.send_update( rule, trigger_warning.alert_threshold + 5, timedelta(minutes=0), subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(), ) self.assert_active_incident(rule) self.assert_slack_calls(["Warning"])