def test(self):
label = "hello"
threshold_type = AlertRuleThresholdType.ABOVE
alert_threshold = 1000
resolve_threshold = 400
trigger = create_alert_rule_trigger(self.alert_rule, label,
threshold_type, alert_threshold,
resolve_threshold)
assert trigger.label == label
assert trigger.threshold_type == threshold_type.value
assert trigger.alert_threshold == alert_threshold
assert trigger.resolve_threshold == resolve_threshold
assert not AlertRuleTriggerExclusion.objects.filter(
alert_rule_trigger=trigger).exists()
def rule(self):
with self.tasks():
rule = self.create_alert_rule(
name="some rule", query="", aggregate="count()", time_window=1, threshold_period=1
)
trigger = create_alert_rule_trigger(
rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10
)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
six.text_type(self.user.id),
)
return rule
def test_excluded_projects(self):
excluded_project = self.create_project(fire_project_created=True)
alert_rule = self.create_alert_rule(
projects=[self.project, excluded_project])
trigger = create_alert_rule_trigger(
alert_rule,
"hi",
AlertRuleThresholdType.ABOVE,
100,
excluded_projects=[excluded_project],
)
# We should have only one exclusion
exclusion = AlertRuleTriggerExclusion.objects.get(
alert_rule_trigger=trigger)
assert exclusion.query_subscription.project == excluded_project
def update(self, instance, validated_data):
with transaction.atomic():
validated_data = self._remove_unchanged_fields(
instance, validated_data)
triggers_data = validated_data.pop("triggers")
alert_rule = update_alert_rule(instance, **validated_data)
# Delete triggers we don't have present in the updated data.
trigger_ids = [x["id"] for x in triggers_data]
AlertRuleTrigger.objects.filter(alert_rule=alert_rule).exclude(
id__in=trigger_ids).delete()
for trigger_data in triggers_data:
actions_data = trigger_data.pop("actions")
try:
if "id" in trigger_data:
trigger_instance = AlertRuleTrigger.objects.get(
alert_rule=alert_rule, id=trigger_data["id"])
trigger_data.pop("id")
trigger = update_alert_rule_trigger(
trigger_instance, **trigger_data)
else:
trigger = create_alert_rule_trigger(
alert_rule=alert_rule, **trigger_data)
except AlertRuleTriggerLabelAlreadyUsedError:
raise serializers.ValidationError(
"This trigger label is already in use for this alert rule"
)
# Delete actions we don't have present in the updated data.
action_ids = [x["id"] for x in actions_data]
AlertRuleTriggerAction.objects.filter(
alert_rule_trigger=trigger).exclude(
id__in=action_ids).delete()
for action_data in actions_data:
if "id" in action_data:
action_instance = AlertRuleTriggerAction.objects.get(
alert_rule_trigger=trigger, id=action_data["id"])
action_data.pop("id")
update_alert_rule_trigger_action(
action_instance, **action_data)
else:
create_alert_rule_trigger_action(trigger=trigger,
**action_data)
return alert_rule
def rule(self):
with self.tasks():
rule = self.create_alert_rule(
name="some rule",
query="",
aggregate="count()",
time_window=1,
threshold_period=1,
resolve_threshold=10,
)
trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
return rule
def rule(self):
rule = create_alert_rule(
self.organization,
[self.project],
"some rule",
query="",
aggregation=QueryAggregations.TOTAL,
time_window=1,
threshold_period=1,
)
trigger = create_alert_rule_trigger(
rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10
)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
six.text_type(self.user.id),
)
return rule
def rule(self):
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
# Make sure the trigger exists
trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
return rule
def test(self):
alert_rule = self.create_alert_rule()
trigger = create_alert_rule_trigger(
alert_rule,
"hi",
AlertRuleThresholdType.ABOVE,
1000,
400,
excluded_projects=[self.project],
)
trigger_id = trigger.id
assert AlertRuleTriggerExclusion.objects.filter(
alert_rule_trigger=trigger,
query_subscription__project=self.project).exists()
delete_alert_rule_trigger(trigger)
assert not AlertRuleTrigger.objects.filter(id=trigger_id).exists()
assert not AlertRuleTriggerExclusion.objects.filter(
alert_rule_trigger=trigger,
query_subscription__project=self.project).exists()
def test(self):
trigger = create_alert_rule_trigger(self.alert_rule, "hello",
AlertRuleThresholdType.ABOVE, 1000,
400)
label = "uh oh"
threshold_type = AlertRuleThresholdType.BELOW
alert_threshold = 2000
resolve_threshold = 800
trigger = update_alert_rule_trigger(
trigger,
label=label,
threshold_type=threshold_type,
alert_threshold=alert_threshold,
resolve_threshold=resolve_threshold,
)
assert trigger.label == label
assert trigger.threshold_type == threshold_type.value
assert trigger.alert_threshold == alert_threshold
assert trigger.resolve_threshold == resolve_threshold
def test_complex_exclude_projects(self):
excluded_project = self.create_project()
other_project = self.create_project(fire_project_created=True)
alert_rule = self.create_alert_rule(
projects=[excluded_project, self.project, other_project]
)
trigger = create_alert_rule_trigger(
alert_rule,
"hi",
AlertRuleThresholdType.ABOVE,
1000,
400,
excluded_projects=[excluded_project, self.project],
)
update_alert_rule_trigger(trigger, excluded_projects=[other_project, excluded_project])
excluded_projects = [
exclusion.query_subscription.project for exclusion in trigger.exclusions.all()
]
assert set(excluded_projects) == set([other_project, excluded_project])
def test_multiple_triggers_at_same_time(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule,
"hello",
AlertRuleThresholdType.ABOVE,
200,
resolve_threshold=50)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER)
processor = self.send_update(rule,
other_trigger.alert_threshold + 1,
timedelta(minutes=-10),
subscription=self.sub)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger,
TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger,
TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident,
[self.action, other_action])
processor = self.send_update(rule,
trigger.resolve_threshold - 1,
timedelta(minutes=-9),
subscription=self.sub)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger,
TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger,
TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident,
[self.action, other_action])
def test_auto_resolve_multiple_trigger(self):
# Test auto resolving works correctly when multiple triggers are present.
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", trigger.alert_threshold - 10)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(rule, other_trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_resolve_separately(self):
# Check that resolve triggers fire separately
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, other_trigger.alert_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [other_action])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def create(self, validated_data):
with transaction.atomic():
try:
# TODO: Remove this, just temporary while we're supporting both fields.
if "aggregation" not in validated_data:
raise serializers.ValidationError(
"aggregation is required")
triggers_data = validated_data.pop("triggers")
# TODO: User super.create and don't duplicate the aggreagation + duplicate name check?
alert_rule = create_alert_rule(
organization=self.context["organization"],
**validated_data)
for trigger_data in triggers_data:
trigger_actions_data = trigger_data.pop("actions")
trigger = create_alert_rule_trigger(alert_rule=alert_rule,
**trigger_data)
for actions_data in trigger_actions_data:
create_alert_rule_trigger_action(trigger=trigger,
**actions_data)
return alert_rule
except AlertRuleNameAlreadyUsedError:
raise serializers.ValidationError(
"This name is already in use for this project")
def create_alert_rule_trigger(alert_rule, label=None, alert_threshold=100):
if not label:
label = petname.Generate(2, " ", letters=10).title()
return create_alert_rule_trigger(alert_rule, label, alert_threshold)
def trigger(self):
return create_alert_rule_trigger(self.alert_rule, "", 100)
def test_existing_label(self):
name = "uh oh"
create_alert_rule_trigger(self.alert_rule, name, AlertRuleThresholdType.ABOVE, 100)
with self.assertRaises(AlertRuleTriggerLabelAlreadyUsedError):
create_alert_rule_trigger(self.alert_rule, name, AlertRuleThresholdType.ABOVE, 100)
def trigger(self):
return create_alert_rule_trigger(
self.alert_rule, "hello", AlertRuleThresholdType.ABOVE, 1000, 400
)
def test(self):
alert_rule = self.create_alert_rule()
trigger = create_alert_rule_trigger(alert_rule, "hi",
AlertRuleThresholdType.ABOVE, 1000,
400)
assert get_triggers_for_alert_rule(alert_rule).get() == trigger
def test_simple(self):
alert_rule = self.create_alert_rule(resolve_threshold=200)
trigger = create_alert_rule_trigger(alert_rule, "hi", 1000)
result = serialize(trigger, serializer=DetailedAlertRuleTriggerSerializer())
self.assert_alert_rule_trigger_serialized(trigger, result)
assert result["excludedProjects"] == []
def test_comparison_above(self):
alert_rule = self.create_alert_rule(comparison_delta=60)
trigger = create_alert_rule_trigger(alert_rule, "hi", 180)
result = serialize(trigger)
self.assert_alert_rule_trigger_serialized(trigger, result, 80)
def test_multiple_triggers(self):
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 1, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(trigger)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_action_handler_called_with_actions(None, [])
# This should cause both to increment, although only `trigger` should fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 1, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_actions_fired_for_incident(incident, [self.action])
# Now only `other_trigger` should increment and fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [other_action])
# Now send through two updates where we're below threshold for `other_trigger`.
# The trigger should end up resolved, but the incident should still be active
processor = self.send_update(
rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [other_action])
# Now we push the other trigger below the resolve threshold twice. This should
# close the incident.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-5), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 1)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-4), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_decimal(self):
alert_rule = self.create_alert_rule(resolve_threshold=200.70)
trigger = create_alert_rule_trigger(alert_rule, "hi", 1000.50)
result = serialize(trigger)
self.assert_alert_rule_trigger_serialized(trigger, result)
def trigger(self):
return create_alert_rule_trigger(self.alert_rule,
CRITICAL_TRIGGER_LABEL, 100)
def test_slack_multiple_triggers_critical_fired_twice_before_warning(self):
"""
Test that ensures that when we get a critical update is sent followed by a warning update,
the warning update is not swallowed and an alert is triggered as a warning alert granted
the count is above the warning trigger threshold
"""
from sentry.incidents.action_handlers import SlackActionHandler
slack_handler = SlackActionHandler
# Create Slack Integration
integration = Integration.objects.create(
provider="slack",
name="Team A",
external_id="TXXXXXXX1",
metadata={
"access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
"installation_type": "born_as_bot",
},
)
integration.add_organization(self.project.organization, self.user)
# Register Slack Handler
AlertRuleTriggerAction.register_type(
"slack",
AlertRuleTriggerAction.Type.SLACK,
[AlertRuleTriggerAction.TargetType.SPECIFIC],
integration_provider="slack",
)(slack_handler)
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule 2",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
trigger = create_alert_rule_trigger(rule, "critical", 100)
trigger_warning = create_alert_rule_trigger(rule, "warning", 10)
for t in [trigger, trigger_warning]:
create_alert_rule_trigger_action(
t,
AlertRuleTriggerAction.Type.SLACK,
AlertRuleTriggerAction.TargetType.SPECIFIC,
integration=integration,
input_channel_id="#workflow",
)
self.assert_slack_calls([])
# Send update above critical
self.send_update(
rule,
trigger.alert_threshold + 5,
timedelta(minutes=-10),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Critical"])
# Send second update above critical
self.send_update(
rule,
trigger.alert_threshold + 6,
timedelta(minutes=-9),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls([])
# Send update below critical but above warning
self.send_update(
rule,
trigger_warning.alert_threshold + 5,
timedelta(minutes=0),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_active_incident(rule)
self.assert_slack_calls(["Warning"])
