class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
def setUp(self):
super(ProcessUpdateTest, self).setUp()
self.old_handlers = AlertRuleTriggerAction._type_registrations
AlertRuleTriggerAction._type_registrations = {}
self.email_action_handler = Mock()
AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])(
self.email_action_handler
)
self._run_tasks = self.tasks()
self._run_tasks.__enter__()
def tearDown(self):
super(ProcessUpdateTest, self).tearDown()
AlertRuleTriggerAction._type_registrations = self.old_handlers
self._run_tasks.__exit__(None, None, None)
@fixture
def other_project(self):
return self.create_project()
@fixture
def sub(self):
return self.rule.snuba_query.subscriptions.filter(project=self.project).get()
@fixture
def other_sub(self):
return self.rule.snuba_query.subscriptions.filter(project=self.other_project).get()
@fixture
def rule(self):
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
# Make sure the trigger exists
trigger = create_alert_rule_trigger(rule, "hi", 100)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
six.text_type(self.user.id),
)
return rule
@fixture
def trigger(self):
return self.rule.alertruletrigger_set.get()
@fixture
def action(self):
return self.trigger.alertruletriggeraction_set.get()
def build_subscription_update(self, subscription, time_delta=None, value=None):
if time_delta is not None:
timestamp = timezone.now() + time_delta
else:
timestamp = timezone.now()
timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0)
data = {}
if subscription:
data = {"some_col_name": randint(0, 100) if value is None else value}
values = {"data": [data]}
return {
"subscription_id": subscription.subscription_id if subscription else uuid4().hex,
"values": values,
"timestamp": timestamp,
"interval": 1,
"partition": 1,
"offset": 1,
}
def send_update(self, rule, value, time_delta=None, subscription=None):
self.email_action_handler.reset_mock()
if time_delta is None:
time_delta = timedelta()
if subscription is None:
subscription = self.sub
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription, value=value, time_delta=time_delta)
with self.feature(["organizations:incidents", "organizations:performance-view"]):
processor.process_update(message)
return processor
def assert_trigger_exists_with_status(self, incident, trigger, status):
assert IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger, status=status.value
).exists()
def assert_trigger_does_not_exist_for_incident(self, incident, trigger):
assert not IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger
).exists()
def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None):
if incidents_to_exclude is None:
incidents_to_exclude = []
assert (
not IncidentTrigger.objects.filter(alert_rule_trigger=trigger)
.exclude(incident__in=incidents_to_exclude)
.exists()
)
def assert_action_handler_called_with_actions(self, incident, actions, project=None):
project = self.project if project is None else project
if not actions:
if not incident:
assert not self.email_action_handler.called
else:
for call_args in self.email_action_handler.call_args_list:
assert call_args[0][1] != incident
else:
self.email_action_handler.assert_has_calls(
[call(action, incident, project) for action in actions], any_order=True
)
def assert_actions_fired_for_incident(self, incident, actions=None, project=None):
actions = [] if actions is None else actions
project = self.project if project is None else project
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list)
def assert_actions_resolved_for_incident(self, incident, actions=None, project=None):
project = self.project if project is None else project
actions = [] if actions is None else actions
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list)
def assert_no_active_incident(self, rule, subscription=None):
assert not self.active_incident_exists(rule, subscription=subscription)
def assert_active_incident(self, rule, subscription=None):
incidents = self.active_incident_exists(rule, subscription=subscription)
assert incidents
return incidents[0]
def active_incident_exists(self, rule, subscription=None):
if subscription is None:
subscription = self.sub
return list(
Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
alert_rule=rule,
projects=subscription.project,
).exclude(status=IncidentStatus.CLOSED.value)
)
def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0):
assert processor.trigger_alert_counts[trigger.id] == alert_triggers
assert processor.rule_resolve_counts == resolve_triggers
alert_stats, resolve_stats = get_alert_rule_stats(
processor.alert_rule, processor.subscription, [trigger]
)[1:]
assert alert_stats[trigger.id] == alert_triggers
assert resolve_stats == resolve_triggers
def test_removed_alert_rule(self):
message = self.build_subscription_update(self.sub)
self.rule.delete()
with self.feature(["organizations:incidents", "organizations:performance-view"]):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription"
)
# TODO: Check subscription is deleted once we start doing that
def test_no_feature(self):
message = self.build_subscription_update(self.sub)
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.ignore_update_missing_incidents"
)
def test_no_feature_performance(self):
self.sub.snuba_query.dataset = "transactions"
message = self.build_subscription_update(self.sub)
with self.feature("organizations:incidents"):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.ignore_update_missing_incidents_performance"
)
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(self.rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
assert incident.date_started == (
timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window)
)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, rule.resolve_threshold - 1)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
def test_auto_resolve(self):
# Verify that we resolve an alert rule automatically even if no resolve
# threshold is set
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_percent_boundary(self):
# Verify that we resolve an alert rule automatically even if no resolve
# threshold is set
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
trigger.update(alert_threshold=0.5)
processor = self.send_update(rule, trigger.alert_threshold + 0.1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_boundary(self):
# Verify that we resolve an alert rule automatically if the value hits the
# original alert trigger value
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_reversed(self):
# Test auto resolving works correctly when threshold is reversed
rule = self.rule
rule.update(resolve_threshold=None, threshold_type=AlertRuleThresholdType.BELOW.value)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_multiple_trigger(self):
# Test auto resolving works correctly when multiple triggers are present.
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", trigger.alert_threshold - 10)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(rule, other_trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
trigger = self.trigger
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
trigger.update(alert_threshold=rule.resolve_threshold + 1)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
trigger = self.trigger
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
trigger.update(alert_threshold=rule.resolve_threshold + 1)
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_subscriptions_do_not_conflict(self):
# Verify that multiple subscriptions associated with a rule don't conflict with
# each other
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
# Send an update through for the first subscription. This shouldn't trigger an
# incident, since we need two consecutive updates that are over the threshold.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Have an update come through for the other sub. This shouldn't influence the original
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Send another update through for the first subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger, [incident])
# Send another update through for the second subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project)
# Now we want to test that resolving is isolated. Send another update through
# for the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
# This second update for the second subscription should resolve its incident,
# but not the incident from the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project)
# This second update for the first subscription should resolve its incident now.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(other_incident, [])
def test_multiple_triggers(self):
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 1, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(trigger)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_action_handler_called_with_actions(None, [])
# This should cause both to increment, although only `trigger` should fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 1, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_actions_fired_for_incident(incident, [self.action])
# Now only `other_trigger` should increment and fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [other_action])
# Now send through two updates where we're below threshold for the rule. This
# should resolve all triggers and the incident should be closed.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 1)
self.assert_trigger_counts(processor, other_trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_at_same_time(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_shutdown(self):
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
valid_wrapper_2 = deepcopy(self.valid_wrapper)
valid_wrapper_2["payload"]["result"]["hello"] = 25
valid_wrapper_3 = deepcopy(valid_wrapper_2)
valid_wrapper_3["payload"]["result"]["hello"] = 5000
self.producer.produce(self.topic, json.dumps(valid_wrapper_2))
self.producer.flush()
counts = [0]
def mock_callback(*args, **kwargs):
counts[0] += 1
if counts[0] > 1:
raise KeyboardInterrupt()
mock = Mock()
mock.side_effect = mock_callback
register_subscriber(self.registration_key)(mock)
sub = QuerySubscription.objects.create(
project=self.project,
type=self.registration_key,
subscription_id=self.subscription_id,
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100)
consumer.run()
valid_payload = self.valid_payload
valid_payload["values"] = valid_payload["result"]
valid_payload["timestamp"] = parse_date(
valid_payload["timestamp"]).replace(tzinfo=pytz.utc)
valid_wrapper_2["payload"]["values"] = valid_wrapper_2["payload"][
"result"]
valid_wrapper_2["payload"]["timestamp"] = parse_date(
valid_wrapper_2["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls(
[call(valid_payload, sub),
call(valid_wrapper_2["payload"], sub)])
# Offset should be committed for the first message, so second run should process
# the second message again
self.producer.produce(self.topic, json.dumps(valid_wrapper_3))
self.producer.flush()
mock.reset_mock()
counts[0] = 0
consumer.run()
valid_wrapper_3["payload"]["values"] = valid_wrapper_3["payload"][
"result"]
valid_wrapper_3["payload"]["timestamp"] = parse_date(
valid_wrapper_3["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls([
call(valid_wrapper_2["payload"], sub),
call(valid_wrapper_3["payload"], sub)
])
