class CreateIncidentTest(TestCase):
record_event = patcher("sentry.analytics.base.Analytics.record_event")
calculate_incident_suspects = patcher("sentry.incidents.tasks.calculate_incident_suspects")
def test_simple(self):
incident_type = IncidentType.CREATED
title = "hello"
query = "goodbye"
date_started = timezone.now()
other_project = self.create_project()
other_group = self.create_group(project=other_project)
self.record_event.reset_mock()
incident = create_incident(
self.organization,
type=incident_type,
title=title,
query=query,
date_started=date_started,
projects=[self.project],
groups=[self.group, other_group],
)
assert incident.identifier == 1
assert incident.status == incident_type.value
assert incident.title == title
assert incident.query == query
assert incident.date_started == date_started
assert incident.date_detected == date_started
assert (
IncidentGroup.objects.filter(
incident=incident, group__in=[self.group, other_group]
).count()
== 2
)
assert (
IncidentProject.objects.filter(
incident=incident, project__in=[self.project, other_project]
).count()
== 2
)
assert (
IncidentActivity.objects.filter(
incident=incident,
type=IncidentActivityType.CREATED.value,
event_stats_snapshot__isnull=False,
).count()
== 1
)
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentCreatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(IncidentType.CREATED.value),
}
self.calculate_incident_suspects.apply_async.assert_called_once_with(
kwargs={"incident_id": incident.id}
)
class UpdateIncidentStatus(TestCase):
record_event = patcher("sentry.analytics.base.Analytics.record_event")
def get_most_recent_incident_activity(self, incident):
return IncidentActivity.objects.filter(incident=incident).order_by("-id")[:1].get()
def test_status_already_set(self):
incident = self.create_incident(status=IncidentStatus.WARNING.value)
update_incident_status(incident, IncidentStatus.WARNING)
assert incident.status == IncidentStatus.WARNING.value
def run_test(self, incident, status, expected_date_closed, user=None, comment=None):
prev_status = incident.status
self.record_event.reset_mock()
update_incident_status(incident, status, user=user, comment=comment)
incident = Incident.objects.get(id=incident.id)
assert incident.status == status.value
assert incident.date_closed == expected_date_closed
activity = self.get_most_recent_incident_activity(incident)
assert activity.type == IncidentActivityType.STATUS_CHANGE.value
assert activity.user == user
if user:
assert IncidentSubscription.objects.filter(incident=incident, user=user).exists()
assert activity.value == six.text_type(status.value)
assert activity.previous_value == six.text_type(prev_status)
assert activity.comment == comment
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentStatusUpdatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(incident.type),
"prev_status": six.text_type(prev_status),
"status": six.text_type(incident.status),
}
def test_closed(self):
incident = create_incident(
self.organization,
IncidentType.ALERT_TRIGGERED,
"Test",
"",
QueryAggregations.TOTAL,
timezone.now(),
projects=[self.project],
)
with self.assertChanges(
lambda: IncidentSnapshot.objects.filter(incident=incident).exists(),
before=False,
after=True,
):
self.run_test(incident, IncidentStatus.CLOSED, timezone.now())
def test_all_params(self):
incident = self.create_incident()
self.run_test(
incident, IncidentStatus.CLOSED, timezone.now(), user=self.user, comment="lol"
)
class BaseSnubaTaskTest(metaclass=abc.ABCMeta):
metrics = patcher("sentry.snuba.tasks.metrics")
status_translations = {
QuerySubscription.Status.CREATING: "create",
QuerySubscription.Status.UPDATING: "update",
QuerySubscription.Status.DELETING: "delete",
}
@abc.abstractproperty
def expected_status(self):
pass
@abc.abstractmethod
def task(self):
pass
def create_subscription(self,
status=None,
subscription_id=None,
dataset=None,
query=None):
if status is None:
status = self.expected_status
if dataset is None:
dataset = QueryDatasets.EVENTS
dataset = dataset.value
aggregate = "count_unique(tags[sentry:user])"
if query is None:
query = "hello"
time_window = 60
resolution = 60
snuba_query = SnubaQuery.objects.create(
dataset=dataset,
aggregate=aggregate,
query=query,
time_window=time_window,
resolution=resolution,
)
return QuerySubscription.objects.create(
snuba_query=snuba_query,
status=status.value,
subscription_id=subscription_id,
project=self.project,
type="something",
)
def test_no_subscription(self):
self.task(12345)
self.metrics.incr.assert_called_once_with(
"snuba.subscriptions.{}.subscription_does_not_exist".format(
self.status_translations[self.expected_status]))
def test_invalid_status(self):
sub = self.create_subscription(QuerySubscription.Status.ACTIVE)
self.task(sub.id)
self.metrics.incr.assert_called_once_with(
"snuba.subscriptions.{}.incorrect_status".format(
self.status_translations[self.expected_status]))
class TestSendSubscriberNotifications(BaseIncidentActivityTest, TestCase):
send_async = patcher("sentry.utils.email.MessageBuilder.send_async")
def test_simple(self):
activity = create_incident_activity(
self.incident, IncidentActivityType.COMMENT, user=self.user, comment="hello"
)
send_subscriber_notifications(activity.id)
# User shouldn't receive an email for their own activity
self.send_async.assert_not_called() # NOQA
self.send_async.reset_mock()
non_member_user = self.create_user(email="*****@*****.**")
subscribe_to_incident(activity.incident, non_member_user)
member_user = self.create_user(email="*****@*****.**")
self.create_member([self.team], user=member_user, organization=self.organization)
subscribe_to_incident(activity.incident, member_user)
send_subscriber_notifications(activity.id)
self.send_async.assert_called_once_with([member_user.email])
assert not IncidentSubscription.objects.filter(
incident=activity.incident, user=non_member_user
).exists()
assert IncidentSubscription.objects.filter(
incident=activity.incident, user=member_user
).exists()
def test_invalid_types(self):
activity_type = IncidentActivityType.CREATED
activity = create_incident_activity(self.incident, activity_type)
send_subscriber_notifications(activity.id)
self.send_async.assert_not_called() # NOQA
self.send_async.reset_mock()
class AlertRuleTriggerActionActivateTest(TestCase):
metrics = patcher("sentry.incidents.models.metrics")
def setUp(self):
self.old_handlers = AlertRuleTriggerAction._type_registrations
AlertRuleTriggerAction._type_registrations = {}
def tearDown(self):
AlertRuleTriggerAction._type_registrations = self.old_handlers
def test_unhandled(self):
trigger = AlertRuleTriggerAction(
type=AlertRuleTriggerAction.Type.EMAIL.value)
trigger.build_handler(Mock(), Mock())
self.metrics.incr.assert_called_once_with(
"alert_rule_trigger.unhandled_type.0")
def test_handled(self):
mock_handler = Mock()
type = AlertRuleTriggerAction.Type.EMAIL
AlertRuleTriggerAction.register_type("something", type,
[])(mock_handler)
trigger = AlertRuleTriggerAction(
type=AlertRuleTriggerAction.Type.EMAIL.value)
incident = Mock()
project = Mock()
trigger.build_handler(incident, project)
mock_handler.assert_called_once_with(trigger, incident, project)
assert not self.metrics.incr.called
class HandleMessageTest(BaseQuerySubscriptionTest, TestCase):
metrics = patcher("sentry.snuba.query_subscription_consumer.metrics")
def test_no_subscription(self):
with mock.patch("sentry.snuba.tasks._snuba_pool") as pool:
pool.urlopen.return_value.status = 202
self.consumer.handle_message(
self.build_mock_message(
self.valid_wrapper, topic=settings.KAFKA_METRICS_SUBSCRIPTIONS_RESULTS
)
)
pool.urlopen.assert_called_once_with(
"DELETE",
"/{}/{}/subscriptions/{}".format(
QueryDatasets.METRICS.value,
EntityKey.MetricsCounters.value,
self.valid_payload["subscription_id"],
),
)
self.metrics.incr.assert_called_once_with(
"snuba_query_subscriber.subscription_doesnt_exist"
)
def test_subscription_not_registered(self):
sub = QuerySubscription.objects.create(
project=self.project, type="unregistered", subscription_id="an_id"
)
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
self.metrics.incr.assert_called_once_with(
"snuba_query_subscriber.subscription_type_not_registered"
)
def test_subscription_registered(self):
registration_key = "registered_test"
mock_callback = mock.Mock()
register_subscriber(registration_key)(mock_callback)
with self.tasks():
snuba_query = create_snuba_query(
QueryDatasets.EVENTS,
"hello",
"count()",
timedelta(minutes=10),
timedelta(minutes=1),
None,
)
sub = create_snuba_subscription(self.project, registration_key, snuba_query)
sub.refresh_from_db()
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
data = deepcopy(data)
data["payload"]["values"] = data["payload"]["result"]
data["payload"]["timestamp"] = parse_date(data["payload"]["timestamp"]).replace(
tzinfo=pytz.utc
)
mock_callback.assert_called_once_with(data["payload"], sub)
class CreateIncidentTest(TestCase):
record_event = patcher("sentry.analytics.base.Analytics.record_event")
def test_simple(self):
incident_type = IncidentType.ALERT_TRIGGERED
title = "hello"
query = "goodbye"
aggregation = QueryAggregations.UNIQUE_USERS
date_started = timezone.now()
other_project = self.create_project(fire_project_created=True)
other_group = self.create_group(project=other_project)
alert_rule = create_alert_rule(
self.organization,
[self.project],
"hello",
"level:error",
QueryAggregations.TOTAL,
10,
1,
)
self.record_event.reset_mock()
incident = create_incident(
self.organization,
type=incident_type,
title=title,
query=query,
aggregation=aggregation,
date_started=date_started,
projects=[self.project],
groups=[self.group, other_group],
alert_rule=alert_rule,
)
assert incident.identifier == 1
assert incident.status == IncidentStatus.OPEN.value
assert incident.type == incident_type.value
assert incident.title == title
assert incident.query == query
assert incident.aggregation == aggregation.value
assert incident.date_started == date_started
assert incident.date_detected == date_started
assert incident.alert_rule == alert_rule
assert (IncidentGroup.objects.filter(
incident=incident, group__in=[self.group,
other_group]).count() == 2)
assert (IncidentProject.objects.filter(
incident=incident, project__in=[self.project,
other_project]).count() == 2)
assert (IncidentActivity.objects.filter(
incident=incident,
type=IncidentActivityType.DETECTED.value).count() == 1)
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentCreatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(IncidentType.ALERT_TRIGGERED.value),
}
class HandleMessageTest(BaseQuerySubscriptionTest, TestCase):
metrics = patcher("sentry.snuba.query_subscription_consumer.metrics")
def test_no_subscription(self):
with mock.patch("sentry.snuba.tasks._snuba_pool") as pool:
pool.urlopen.return_value.status = 202
self.consumer.handle_message(
self.build_mock_message(
self.valid_wrapper,
topic=settings.KAFKA_EVENTS_SUBSCRIPTIONS_RESULTS))
pool.urlopen.assert_called_once_with(
"DELETE",
"/{}/subscriptions/{}".format(
QueryDatasets.EVENTS.value,
self.valid_payload["subscription_id"]),
)
self.metrics.incr.assert_called_once_with(
"snuba_query_subscriber.subscription_doesnt_exist")
def test_subscription_not_registered(self):
sub = QuerySubscription.objects.create(
project=self.project,
type="unregistered",
subscription_id="an_id",
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
self.metrics.incr.assert_called_once_with(
"snuba_query_subscriber.subscription_type_not_registered")
def test_subscription_registered(self):
registration_key = "registered_test"
mock_callback = Mock()
register_subscriber(registration_key)(mock_callback)
sub = QuerySubscription.objects.create(
project=self.project,
type=registration_key,
subscription_id="an_id",
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
data = deepcopy(data)
data["payload"]["values"] = data["payload"]["result"]
data["payload"]["timestamp"] = parse_date(
data["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock_callback.assert_called_once_with(data["payload"], sub)
class HandleTriggerActionTest(TestCase):
metrics = patcher("sentry.incidents.tasks.metrics")
@fixture
def alert_rule(self):
return self.create_alert_rule()
@fixture
def trigger(self):
return create_alert_rule_trigger(self.alert_rule,
CRITICAL_TRIGGER_LABEL, 100)
@fixture
def action(self):
return create_alert_rule_trigger_action(
self.trigger, AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER)
def test_missing_trigger_action(self):
with self.tasks():
handle_trigger_action.delay(1000, 1001, self.project.id, "hello")
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.action.skipping_missing_action")
def test_missing_incident(self):
with self.tasks():
handle_trigger_action.delay(self.action.id, 1001, self.project.id,
"hello")
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.action.skipping_missing_incident")
def test_missing_project(self):
incident = self.create_incident()
with self.tasks():
handle_trigger_action.delay(self.action.id, incident.id, 1002,
"hello")
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.action.skipping_missing_project")
def test(self):
with patch.object(AlertRuleTriggerAction,
"_type_registrations",
new={}):
mock_handler = Mock()
AlertRuleTriggerAction.register_type(
"email", AlertRuleTriggerAction.Type.EMAIL, [])(mock_handler)
incident = self.create_incident()
metric_value = 1234
with self.tasks():
handle_trigger_action.delay(self.action.id,
incident.id,
self.project.id,
"fire",
metric_value=metric_value)
mock_handler.assert_called_once_with(self.action, incident,
self.project)
mock_handler.return_value.fire.assert_called_once_with(
metric_value, IncidentStatus.CRITICAL)
class HandleTriggerActionTest(TestCase):
metrics = patcher("sentry.incidents.tasks.metrics")
@fixture
def alert_rule(self):
return self.create_alert_rule()
@fixture
def trigger(self):
return create_alert_rule_trigger(self.alert_rule, "",
AlertRuleThresholdType.ABOVE, 100)
@fixture
def action(self):
return create_alert_rule_trigger_action(
self.trigger, AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER)
def test_missing_trigger_action(self):
with self.tasks():
handle_trigger_action.delay(1000, 1001, self.project.id, "hello")
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_missing_action")
def test_missing_incident(self):
with self.tasks():
handle_trigger_action.delay(self.action.id, 1001, self.project.id,
"hello")
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_missing_incident")
def test_missing_project(self):
incident = self.create_incident()
with self.tasks():
handle_trigger_action.delay(self.action.id, incident.id, 1002,
"hello")
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_missing_project")
def test(self):
with patch.object(AlertRuleTriggerAction, "handlers", new={}):
mock_handler = Mock()
AlertRuleTriggerAction.register_type_handler(
AlertRuleTriggerAction.Type.EMAIL)(mock_handler)
incident = self.create_incident()
with self.tasks():
handle_trigger_action.delay(self.action.id, incident.id,
self.project.id, "fire")
mock_handler.assert_called_once_with(self.action, incident,
self.project)
mock_handler.return_value.fire.assert_called_once_with()
class HandleMessageTest(BaseQuerySubscriptionTest, TestCase):
metrics = patcher("sentry.snuba.query_subscription_consumer.metrics")
def test_no_subscription(self):
self.consumer.handle_message(self.build_mock_message(self.valid_wrapper))
self.metrics.incr.assert_called_once_with(
"snuba_query_subscriber.subscription_doesnt_exist"
)
def test_subscription_not_registered(self):
sub = QuerySubscription.objects.create(
project=self.project,
type="unregistered",
subscription_id="an_id",
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
self.metrics.incr.assert_called_once_with(
"snuba_query_subscriber.subscription_type_not_registered"
)
def test_subscription_registered(self):
registration_key = "registered_test"
mock_callback = Mock()
register_subscriber(registration_key)(mock_callback)
sub = QuerySubscription.objects.create(
project=self.project,
type=registration_key,
subscription_id="an_id",
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
data = self.valid_wrapper
data["payload"]["subscription_id"] = sub.subscription_id
self.consumer.handle_message(self.build_mock_message(data))
data["payload"]["timestamp"] = parse_date(data["payload"]["timestamp"]).replace(
tzinfo=pytz.utc
)
mock_callback.assert_called_once_with(data["payload"], sub)
class BaseSnubaTaskTest(object):
metrics = patcher("sentry.snuba.tasks.metrics")
status_translations = {
QuerySubscription.Status.CREATING: "create",
QuerySubscription.Status.UPDATING: "update",
QuerySubscription.Status.DELETING: "delete",
}
@abc.abstractproperty
def expected_status(self):
pass
@abc.abstractmethod
def task(self):
pass
def create_subscription(self, status=None, subscription_id=None):
if status is None:
status = self.expected_status
return QuerySubscription.objects.create(
status=status.value,
subscription_id=subscription_id,
project=self.project,
type="something",
dataset=QueryDatasets.EVENTS.value,
query="hello",
aggregation=QueryAggregations.UNIQUE_USERS.value,
time_window=60,
resolution=60,
)
def test_no_subscription(self):
self.task(12345)
self.metrics.incr.assert_called_once_with(
"snuba.subscriptions.{}.subscription_does_not_exist".format(
self.status_translations[self.expected_status]
)
)
def test_invalid_status(self):
sub = self.create_subscription(QuerySubscription.Status.ACTIVE)
self.task(sub.id)
self.metrics.incr.assert_called_once_with(
"snuba.subscriptions.{}.incorrect_status".format(
self.status_translations[self.expected_status]
)
)
class SwitchFormTest(Exam, unittest2.TestCase):
mock_switch = fixture(Mock, conditions=[1, 2, 3])
condition_form = patcher('gutter.django.forms.ConditionForm')
form = fixture(SwitchForm)
@fixture
def switch_from_object(self):
return SwitchForm.from_object(self.mock_switch)
@patch('gutter.django.forms.ConditionFormSet')
def test_from_object_returns_dict_of_properties(self, _):
eq_(
self.switch_from_object.initial,
dict(
label=self.mock_switch.label,
name=self.mock_switch.name,
description=self.mock_switch.description,
state=self.mock_switch.state,
compounded=self.mock_switch.compounded,
concent=self.mock_switch.concent,
))
@patch('gutter.django.forms.ConditionFormSet')
def test_from_object_sets_conditions_as_form_set(self, ConditionFormSet):
eq_(self.switch_from_object.conditions, ConditionFormSet.return_value)
# Called with a map() over ConditionForm.to_dict
expected = [self.condition_form.to_dict.return_value] * 3
ConditionFormSet.assert_called_once_with(initial=expected)
# Assert that the calls it did receive are correct
self.condition_form.to_dict.assert_any_call(1)
self.condition_form.to_dict.assert_any_call(2)
self.condition_form.to_dict.assert_any_call(3)
def test_from_object_marks_the_name_field_as_readonly(self):
self.assertTrue(
self.switch_from_object.fields['name'].widget.attrs['readonly'])
class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
@fixture
def other_project(self):
return self.create_project()
@fixture
def sub(self):
return self.rule.query_subscriptions.filter(project=self.project).get()
@fixture
def other_sub(self):
return self.rule.query_subscriptions.filter(project=self.other_project).get()
@fixture
def rule(self):
rule = create_alert_rule(
self.organization,
[self.project, self.other_project],
"some rule",
AlertRuleThresholdType.ABOVE,
query="",
aggregation=QueryAggregations.TOTAL,
time_window=1,
alert_threshold=100,
resolve_threshold=10,
threshold_period=1,
)
return rule
def build_subscription_update(self, subscription, time_delta=None, value=None):
if time_delta is not None:
timestamp = int(to_timestamp(timezone.now() + time_delta))
else:
timestamp = int(time())
values = {}
if subscription:
aggregation_type = query_aggregation_to_snuba[
QueryAggregations(subscription.aggregation)
]
value = randint(0, 100) if value is None else value
values = {aggregation_type[2]: value}
return {
"subscription_id": subscription.subscription_id if subscription else uuid4().hex,
"values": values,
"timestamp": timestamp,
"interval": 1,
"partition": 1,
"offset": 1,
}
def send_update(self, rule, value, time_delta=None, subscription=None):
if time_delta is None:
time_delta = timedelta()
if subscription is None:
subscription = self.sub
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription, value=value, time_delta=time_delta)
processor.process_update(message)
return processor
def assert_no_active_incident(self, rule, subscription=None):
assert not self.active_incident_exists(rule, subscription=subscription)
def assert_active_incident(self, rule, subscription=None):
assert self.active_incident_exists(rule, subscription=subscription)
def active_incident_exists(self, rule, subscription=None):
if subscription is None:
subscription = self.sub
return Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
status=IncidentStatus.OPEN.value,
alert_rule=rule,
projects=subscription.project,
).exists()
def assert_trigger_counts(self, processor, alert_triggers=0, resolve_triggers=0):
assert processor.alert_triggers == alert_triggers
assert processor.resolve_triggers == resolve_triggers
assert get_alert_rule_stats(processor.alert_rule, processor.subscription)[1:] == (
alert_triggers,
resolve_triggers,
)
def test_removed_alert_rule(self):
message = self.build_subscription_update(self.sub)
self.rule.delete()
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription"
)
# TODO: Check subscription is deleted once we start doing that
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.rule.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.rule.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.rule.alert_threshold, timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.rule.alert_threshold, timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold)
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(self.rule)
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1)
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
def test_alert_multiple_triggers(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule)
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
processor = self.send_update(rule, rule.resolve_threshold - 1)
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_resolve_multiple_triggers(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_resolve_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-4))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule)
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, rule.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, rule.alert_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_multiple_subscriptions_do_not_conflict(self):
# Verify that multiple subscriptions associated with a rule don't conflict with
# each other
rule = self.rule
rule.update(threshold_period=2)
# Send an update through for the first subscription. This shouldn't trigger an
# incident, since we need two consecutive updates that are over the threshold.
processor = self.send_update(
rule, rule.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule, self.sub)
# Have an update come through for the other sub. This shouldn't influence the original
processor = self.send_update(
rule, rule.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub
)
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_no_active_incident(rule, self.other_sub)
# Send another update through for the first subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, rule.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule, self.sub)
self.assert_no_active_incident(rule, self.other_sub)
# Send another update through for the second subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, rule.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub
)
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule, self.sub)
self.assert_active_incident(rule, self.other_sub)
# Now we want to test that resolving is isolated. Send another update through
# for the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule, self.sub)
self.assert_active_incident(rule, self.other_sub)
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub
)
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule, self.sub)
self.assert_active_incident(rule, self.other_sub)
# This second update for the second subscription should resolve its incident,
# but not the incident from the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub
)
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule, self.sub)
self.assert_no_active_incident(rule, self.other_sub)
# This second update for the first subscription should resolve its incident now.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_no_active_incident(rule, self.other_sub)
class CreateIncidentActivityTest(TestCase, BaseIncidentsTest):
send_subscriber_notifications = patcher(
"sentry.incidents.tasks.send_subscriber_notifications")
record_event = patcher("sentry.analytics.base.Analytics.record_event")
def assert_notifications_sent(self, activity):
self.send_subscriber_notifications.apply_async.assert_called_once_with(
kwargs={"activity_id": activity.id}, countdown=10)
def test_no_snapshot(self):
incident = self.create_incident()
self.record_event.reset_mock()
activity = create_incident_activity(
incident,
IncidentActivityType.STATUS_CHANGE,
user=self.user,
value=six.text_type(IncidentStatus.CLOSED.value),
previous_value=six.text_type(IncidentStatus.WARNING.value),
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.STATUS_CHANGE.value
assert activity.user == self.user
assert activity.value == six.text_type(IncidentStatus.CLOSED.value)
assert activity.previous_value == six.text_type(
IncidentStatus.WARNING.value)
self.assert_notifications_sent(activity)
assert not self.record_event.called
def test_comment(self):
incident = self.create_incident()
comment = "hello"
with self.assertChanges(
lambda: IncidentSubscription.objects.filter(
incident=incident, user=self.user).exists(),
before=False,
after=True,
):
self.record_event.reset_mock()
activity = create_incident_activity(incident,
IncidentActivityType.COMMENT,
user=self.user,
comment=comment)
assert activity.incident == incident
assert activity.type == IncidentActivityType.COMMENT.value
assert activity.user == self.user
assert activity.comment == comment
assert activity.value is None
assert activity.previous_value is None
self.assert_notifications_sent(activity)
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentCommentCreatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(incident.type),
"user_id": six.text_type(self.user.id),
"activity_id": six.text_type(activity.id),
}
def test_mentioned_user_ids(self):
incident = self.create_incident()
mentioned_member = self.create_user()
subscribed_mentioned_member = self.create_user()
IncidentSubscription.objects.create(incident=incident,
user=subscribed_mentioned_member)
comment = "hello **@%s** and **@%s**" % (
mentioned_member.username,
subscribed_mentioned_member.username,
)
with self.assertChanges(
lambda: IncidentSubscription.objects.filter(
incident=incident, user=mentioned_member).exists(),
before=False,
after=True,
):
self.record_event.reset_mock()
activity = create_incident_activity(
incident,
IncidentActivityType.COMMENT,
user=self.user,
comment=comment,
mentioned_user_ids=[
mentioned_member.id, subscribed_mentioned_member.id
],
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.COMMENT.value
assert activity.user == self.user
assert activity.comment == comment
assert activity.value is None
assert activity.previous_value is None
self.assert_notifications_sent(activity)
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentCommentCreatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(incident.type),
"user_id": six.text_type(self.user.id),
"activity_id": six.text_type(activity.id),
}
class CreateIncidentActivityTest(TestCase, BaseIncidentsTest):
send_subscriber_notifications = patcher(
'sentry.incidents.logic.send_subscriber_notifications')
def assert_notifications_sent(self, activity):
self.send_subscriber_notifications.apply_async.assert_called_once_with(
kwargs={'activity_id': activity.id},
countdown=10,
)
def test_no_snapshot(self):
incident = self.create_incident()
activity = create_incident_activity(
incident,
IncidentActivityType.STATUS_CHANGE,
user=self.user,
value=six.text_type(IncidentStatus.CLOSED.value),
previous_value=six.text_type(IncidentStatus.OPEN.value),
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.STATUS_CHANGE.value
assert activity.user == self.user
assert activity.value == six.text_type(IncidentStatus.CLOSED.value)
assert activity.previous_value == six.text_type(
IncidentStatus.OPEN.value)
self.assert_notifications_sent(activity)
def test_snapshot(self):
self.create_event(self.now - timedelta(minutes=2))
self.create_event(self.now - timedelta(minutes=2))
self.create_event(self.now - timedelta(minutes=1))
# Define events outside incident range. Should be included in the
# snapshot
self.create_event(self.now - timedelta(minutes=20))
self.create_event(self.now - timedelta(minutes=30))
# Too far out, should be excluded
self.create_event(self.now - timedelta(minutes=100))
incident = self.create_incident(date_started=self.now -
timedelta(minutes=5),
query='',
projects=[self.project])
event_stats_snapshot = create_initial_event_stats_snapshot(incident)
activity = create_incident_activity(
incident,
IncidentActivityType.CREATED,
event_stats_snapshot=event_stats_snapshot,
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.CREATED.value
assert activity.value is None
assert activity.previous_value is None
assert event_stats_snapshot == activity.event_stats_snapshot
self.assert_notifications_sent(activity)
def test_comment(self):
incident = self.create_incident()
comment = 'hello'
with self.assertChanges(
lambda: IncidentSubscription.objects.filter(
incident=incident,
user=self.user,
).exists(),
before=False,
after=True,
):
activity = create_incident_activity(
incident,
IncidentActivityType.COMMENT,
user=self.user,
comment=comment,
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.COMMENT.value
assert activity.user == self.user
assert activity.comment == comment
assert activity.value is None
assert activity.previous_value is None
self.assert_notifications_sent(activity)
class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
def setUp(self):
super(ProcessUpdateTest, self).setUp()
self.old_handlers = AlertRuleTriggerAction._type_registrations
AlertRuleTriggerAction._type_registrations = {}
self.email_action_handler = Mock()
AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])(
self.email_action_handler
)
self._run_tasks = self.tasks()
self._run_tasks.__enter__()
def tearDown(self):
super(ProcessUpdateTest, self).tearDown()
AlertRuleTriggerAction._type_registrations = self.old_handlers
self._run_tasks.__exit__(None, None, None)
@fixture
def other_project(self):
return self.create_project()
@fixture
def sub(self):
return self.rule.query_subscriptions.filter(project=self.project).get()
@fixture
def other_sub(self):
return self.rule.query_subscriptions.filter(project=self.other_project).get()
@fixture
def rule(self):
rule = create_alert_rule(
self.organization,
[self.project, self.other_project],
"some rule",
query="",
aggregation=QueryAggregations.TOTAL,
time_window=1,
threshold_period=1,
)
# Make sure the trigger exists
trigger = create_alert_rule_trigger(
rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10
)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
six.text_type(self.user.id),
)
return rule
@fixture
def trigger(self):
return self.rule.alertruletrigger_set.get()
@fixture
def action(self):
return self.trigger.alertruletriggeraction_set.get()
def build_subscription_update(self, subscription, time_delta=None, value=None):
if time_delta is not None:
timestamp = timezone.now() + time_delta
else:
timestamp = timezone.now()
timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0)
data = {}
if subscription:
aggregation_type = query_aggregation_to_snuba[
QueryAggregations(subscription.aggregation)
]
value = randint(0, 100) if value is None else value
data = {aggregation_type[2]: value}
values = {"data": [data]}
return {
"subscription_id": subscription.subscription_id if subscription else uuid4().hex,
"values": values,
"timestamp": timestamp,
"interval": 1,
"partition": 1,
"offset": 1,
}
def send_update(self, rule, value, time_delta=None, subscription=None):
self.email_action_handler.reset_mock()
if time_delta is None:
time_delta = timedelta()
if subscription is None:
subscription = self.sub
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription, value=value, time_delta=time_delta)
processor.process_update(message)
return processor
def assert_trigger_exists_with_status(self, incident, trigger, status):
assert IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger, status=status.value
).exists()
def assert_trigger_does_not_exist_for_incident(self, incident, trigger):
assert not IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger
).exists()
def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None):
if incidents_to_exclude is None:
incidents_to_exclude = []
assert (
not IncidentTrigger.objects.filter(alert_rule_trigger=trigger)
.exclude(incident__in=incidents_to_exclude)
.exists()
)
def assert_action_handler_called_with_actions(self, incident, actions, project=None):
project = self.project if project is None else project
if not actions:
if not incident:
assert not self.email_action_handler.called
else:
for call_args in self.email_action_handler.call_args_list:
assert call_args[0][1] != incident
else:
self.email_action_handler.assert_has_calls(
[call(action, incident, project) for action in actions], any_order=True
)
def assert_actions_fired_for_incident(self, incident, actions=None, project=None):
actions = [] if actions is None else actions
project = self.project if project is None else project
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list)
def assert_actions_resolved_for_incident(self, incident, actions=None, project=None):
project = self.project if project is None else project
actions = [] if actions is None else actions
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list)
def assert_no_active_incident(self, rule, subscription=None):
assert not self.active_incident_exists(rule, subscription=subscription)
def assert_active_incident(self, rule, subscription=None):
incidents = self.active_incident_exists(rule, subscription=subscription)
assert incidents
return incidents[0]
def active_incident_exists(self, rule, subscription=None):
if subscription is None:
subscription = self.sub
return list(
Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
alert_rule=rule,
projects=subscription.project,
).exclude(status=IncidentStatus.CLOSED.value)
)
def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0):
assert processor.trigger_alert_counts[trigger.id] == alert_triggers
assert processor.trigger_resolve_counts[trigger.id] == resolve_triggers
alert_stats, resolve_stats = get_alert_rule_stats(
processor.alert_rule, processor.subscription, [trigger]
)[1:]
assert alert_stats[trigger.id] == alert_triggers
assert resolve_stats[trigger.id] == resolve_triggers
def test_removed_alert_rule(self):
message = self.build_subscription_update(self.sub)
self.rule.delete()
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription"
)
# TODO: Check subscription is deleted once we start doing that
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(self.rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.resolve_threshold - 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
trigger = self.trigger
trigger.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
trigger = self.trigger
trigger.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_subscriptions_do_not_conflict(self):
# Verify that multiple subscriptions associated with a rule don't conflict with
# each other
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
# Send an update through for the first subscription. This shouldn't trigger an
# incident, since we need two consecutive updates that are over the threshold.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Have an update come through for the other sub. This shouldn't influence the original
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Send another update through for the first subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger, [incident])
# Send another update through for the second subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project)
# Now we want to test that resolving is isolated. Send another update through
# for the first subscription.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
# This second update for the second subscription should resolve its incident,
# but not the incident from the first subscription.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project)
# This second update for the first subscription should resolve its incident now.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(other_incident, [])
def test_multiple_triggers(self):
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 1, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(trigger)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_action_handler_called_with_actions(None, [])
# This should cause both to increment, although only `trigger` should fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 1, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_actions_fired_for_incident(incident, [self.action])
# Now only `other_trigger` should increment and fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [other_action])
# Now send through two updates where we're below threshold for `other_trigger`.
# The trigger should end up resolved, but the incident should still be active
processor = self.send_update(
rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [other_action])
# Now we push the other trigger below the resolve threshold twice. This should
# close the incident.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-5), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 1)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-4), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_triggers_at_same_time(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_one_with_no_resolve(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_resolved_for_incident(incident, [self.action])
class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
slack_client = patcher("sentry.integrations.slack.utils.SlackClient.post")
def setUp(self):
super().setUp()
self.old_handlers = AlertRuleTriggerAction._type_registrations
AlertRuleTriggerAction._type_registrations = {}
self.email_action_handler = Mock()
AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])(
self.email_action_handler
)
self._run_tasks = self.tasks()
self._run_tasks.__enter__()
def tearDown(self):
super().tearDown()
AlertRuleTriggerAction._type_registrations = self.old_handlers
self._run_tasks.__exit__(None, None, None)
@fixture
def other_project(self):
return self.create_project()
@fixture
def sub(self):
return self.rule.snuba_query.subscriptions.filter(project=self.project).get()
@fixture
def other_sub(self):
return self.rule.snuba_query.subscriptions.filter(project=self.other_project).get()
@fixture
def rule(self):
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
# Make sure the trigger exists
trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
return rule
@fixture
def trigger(self):
return self.rule.alertruletrigger_set.get()
@fixture
def action(self):
return self.trigger.alertruletriggeraction_set.get()
def build_subscription_update(self, subscription, time_delta=None, value=EMPTY):
if time_delta is not None:
timestamp = timezone.now() + time_delta
else:
timestamp = timezone.now()
timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0)
data = {}
if subscription:
data = {"some_col_name": randint(0, 100) if value is EMPTY else value}
values = {"data": [data]}
return {
"subscription_id": subscription.subscription_id if subscription else uuid4().hex,
"values": values,
"timestamp": timestamp,
"interval": 1,
"partition": 1,
"offset": 1,
}
def send_update(self, rule, value, time_delta=None, subscription=None):
self.email_action_handler.reset_mock()
if time_delta is None:
time_delta = timedelta()
if subscription is None:
subscription = self.sub
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription, value=value, time_delta=time_delta)
with self.feature(
["organizations:incidents", "organizations:performance-view"]
), self.capture_on_commit_callbacks(execute=True):
processor.process_update(message)
return processor
def assert_slack_calls(self, trigger_labels):
expected = [f"{label}: some rule 2" for label in trigger_labels]
actual = [
json.loads(call_kwargs["data"]["attachments"])[0]["title"]
for (_, call_kwargs) in self.slack_client.call_args_list
]
assert expected == actual
self.slack_client.reset_mock()
def assert_trigger_exists_with_status(self, incident, trigger, status):
assert IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger, status=status.value
).exists()
def assert_trigger_does_not_exist_for_incident(self, incident, trigger):
assert not IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger
).exists()
def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None):
if incidents_to_exclude is None:
incidents_to_exclude = []
assert (
not IncidentTrigger.objects.filter(alert_rule_trigger=trigger)
.exclude(incident__in=incidents_to_exclude)
.exists()
)
def assert_action_handler_called_with_actions(self, incident, actions, project=None):
project = self.project if project is None else project
if not actions:
if not incident:
assert not self.email_action_handler.called
else:
for call_args in self.email_action_handler.call_args_list:
assert call_args[0][1] != incident
else:
self.email_action_handler.assert_has_calls(
[call(action, incident, project) for action in actions], any_order=True
)
def assert_actions_fired_for_incident(self, incident, actions=None, project=None):
actions = [] if actions is None else actions
project = self.project if project is None else project
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list)
def assert_actions_resolved_for_incident(self, incident, actions=None, project=None):
project = self.project if project is None else project
actions = [] if actions is None else actions
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list)
def assert_no_active_incident(self, rule, subscription=None):
assert not self.active_incident_exists(rule, subscription=subscription)
def assert_active_incident(self, rule, subscription=None):
incidents = self.active_incident_exists(rule, subscription=subscription)
assert incidents
return incidents[0]
def active_incident_exists(self, rule, subscription=None):
if subscription is None:
subscription = self.sub
return list(
Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
alert_rule=rule,
projects=subscription.project,
).exclude(status=IncidentStatus.CLOSED.value)
)
def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0):
assert processor.trigger_alert_counts[trigger.id] == alert_triggers
alert_stats, resolve_stats = get_alert_rule_stats(
processor.alert_rule, processor.subscription, [trigger]
)[1:]
assert alert_stats[trigger.id] == alert_triggers
assert resolve_stats[trigger.id] == resolve_triggers
def test_removed_alert_rule(self):
message = self.build_subscription_update(self.sub)
self.rule.delete()
with self.feature(["organizations:incidents", "organizations:performance-view"]):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription"
)
# TODO: Check subscription is deleted once we start doing that
def test_removed_project(self):
message = self.build_subscription_update(self.sub)
self.project.delete()
with self.feature(["organizations:incidents", "organizations:performance-view"]):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with("incidents.alert_rules.ignore_deleted_project")
def test_no_feature(self):
message = self.build_subscription_update(self.sub)
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.ignore_update_missing_incidents"
)
def test_no_feature_performance(self):
self.sub.snuba_query.dataset = "transactions"
message = self.build_subscription_update(self.sub)
with self.feature("organizations:incidents"):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.ignore_update_missing_incidents_performance"
)
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(self.rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
assert incident.date_started == (
timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window)
)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_dedupe(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
c_trigger = self.trigger
c_action_2 = create_alert_rule_trigger_action(
self.trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
w_trigger = create_alert_rule_trigger(
self.rule, WARNING_TRIGGER_LABEL, c_trigger.alert_threshold - 10
)
create_alert_rule_trigger_action(
w_trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
processor = self.send_update(rule, c_trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
assert incident.date_started == (
timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window)
)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [c_action_2])
def test_alert_nullable(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
self.trigger
processor = self.send_update(rule, None)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
def test_alert_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, rule.resolve_threshold - 1)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
def test_auto_resolve(self):
# Verify that we resolve an alert rule automatically even if no resolve
# threshold is set
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_percent_boundary(self):
# Verify that we resolve an alert rule automatically even if no resolve
# threshold is set
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
trigger.update(alert_threshold=0.5)
processor = self.send_update(rule, trigger.alert_threshold + 0.1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_boundary(self):
# Verify that we resolve an alert rule automatically if the value hits the
# original alert trigger value
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_reversed(self):
# Test auto resolving works correctly when threshold is reversed
rule = self.rule
rule.update(resolve_threshold=None, threshold_type=AlertRuleThresholdType.BELOW.value)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_multiple_trigger(self):
# Test auto resolving works correctly when multiple triggers are present.
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", trigger.alert_threshold - 10)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(rule, other_trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
trigger = self.trigger
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
trigger.update(alert_threshold=rule.resolve_threshold + 1)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
trigger = self.trigger
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
trigger.update(alert_threshold=rule.resolve_threshold + 1)
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_subscriptions_do_not_conflict(self):
# Verify that multiple subscriptions associated with a rule don't conflict with
# each other
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
# Send an update through for the first subscription. This shouldn't trigger an
# incident, since we need two consecutive updates that are over the threshold.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Have an update come through for the other sub. This shouldn't influence the original
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Send another update through for the first subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger, [incident])
# Send another update through for the second subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project)
# Now we want to test that resolving is isolated. Send another update through
# for the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
# This second update for the second subscription should resolve its incident,
# but not the incident from the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project)
# This second update for the first subscription should resolve its incident now.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(other_incident, [])
def test_multiple_triggers(self):
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 1, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(trigger)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_action_handler_called_with_actions(None, [])
# This should cause both to increment, although only `trigger` should fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 1, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_actions_fired_for_incident(incident, [self.action])
# Now only `other_trigger` should increment and fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [other_action])
# Now send through two updates where we're below threshold for the rule. This
# should resolve all triggers and the incident should be closed.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 1)
self.assert_trigger_counts(processor, other_trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_slack_multiple_triggers_critical_before_warning(self):
"""
Test that ensures that when we get a critical update is sent followed by a warning update,
the warning update is not swallowed and an alert is triggered as a warning alert granted
the count is above the warning trigger threshold
"""
from sentry.incidents.action_handlers import SlackActionHandler
slack_handler = SlackActionHandler
# Create Slack Integration
integration = Integration.objects.create(
provider="slack",
name="Team A",
external_id="TXXXXXXX1",
metadata={
"access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
"installation_type": "born_as_bot",
},
)
integration.add_organization(self.project.organization, self.user)
# Register Slack Handler
AlertRuleTriggerAction.register_type(
"slack",
AlertRuleTriggerAction.Type.SLACK,
[AlertRuleTriggerAction.TargetType.SPECIFIC],
integration_provider="slack",
)(slack_handler)
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule 2",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
trigger = create_alert_rule_trigger(rule, "critical", 100)
trigger_warning = create_alert_rule_trigger(rule, "warning", 10)
for t in [trigger, trigger_warning]:
create_alert_rule_trigger_action(
t,
AlertRuleTriggerAction.Type.SLACK,
AlertRuleTriggerAction.TargetType.SPECIFIC,
integration=integration,
input_channel_id="#workflow",
)
# Send Critical Update
self.send_update(
rule,
trigger.alert_threshold + 5,
timedelta(minutes=-10),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Critical"])
# Send Warning Update
self.send_update(
rule,
trigger_warning.alert_threshold + 5,
timedelta(minutes=0),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Warning"])
self.assert_active_incident(rule)
def test_slack_multiple_triggers_critical_fired_twice_before_warning(self):
"""
Test that ensures that when we get a critical update is sent followed by a warning update,
the warning update is not swallowed and an alert is triggered as a warning alert granted
the count is above the warning trigger threshold
"""
from sentry.incidents.action_handlers import SlackActionHandler
slack_handler = SlackActionHandler
# Create Slack Integration
integration = Integration.objects.create(
provider="slack",
name="Team A",
external_id="TXXXXXXX1",
metadata={
"access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
"installation_type": "born_as_bot",
},
)
integration.add_organization(self.project.organization, self.user)
# Register Slack Handler
AlertRuleTriggerAction.register_type(
"slack",
AlertRuleTriggerAction.Type.SLACK,
[AlertRuleTriggerAction.TargetType.SPECIFIC],
integration_provider="slack",
)(slack_handler)
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule 2",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
trigger = create_alert_rule_trigger(rule, "critical", 100)
trigger_warning = create_alert_rule_trigger(rule, "warning", 10)
for t in [trigger, trigger_warning]:
create_alert_rule_trigger_action(
t,
AlertRuleTriggerAction.Type.SLACK,
AlertRuleTriggerAction.TargetType.SPECIFIC,
integration=integration,
input_channel_id="#workflow",
)
self.assert_slack_calls([])
# Send update above critical
self.send_update(
rule,
trigger.alert_threshold + 5,
timedelta(minutes=-10),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Critical"])
# Send second update above critical
self.send_update(
rule,
trigger.alert_threshold + 6,
timedelta(minutes=-9),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls([])
# Send update below critical but above warning
self.send_update(
rule,
trigger_warning.alert_threshold + 5,
timedelta(minutes=0),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_active_incident(rule)
self.assert_slack_calls(["Warning"])
def test_multiple_triggers_at_same_time(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_resolve_separately(self):
# Check that resolve triggers fire separately
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, other_trigger.alert_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [other_action])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
@fixture
def subscription(self):
subscription = QuerySubscription.objects.create(
project=self.project,
type=INCIDENTS_SNUBA_SUBSCRIPTION_TYPE,
subscription_id="some_id",
dataset=SnubaDatasets.EVENTS.value,
query="",
aggregations=[AlertRuleAggregations.TOTAL.value],
time_window=1,
resolution=1,
)
return subscription
@fixture
def rule(self):
rule = create_alert_rule(
self.project,
"some rule",
AlertRuleThresholdType.ABOVE,
query="",
aggregations=[AlertRuleAggregations.TOTAL],
time_window=1,
alert_threshold=100,
resolve_threshold=10,
threshold_period=1,
)
rule.update(query_subscription=self.subscription)
return rule
def build_subscription_update(self,
subscription=None,
time_delta=None,
value=None):
if time_delta is not None:
timestamp = int(to_timestamp(timezone.now() + time_delta))
else:
timestamp = int(time())
values = {}
if subscription:
aggregation_type = alert_aggregation_to_snuba[
AlertRuleAggregations(subscription.aggregations[0])]
value = randint(0, 100) if value is None else value
values = {aggregation_type[2]: value}
return {
"subscription_id":
subscription.subscription_id if subscription else uuid4().hex,
"values":
values,
"timestamp":
timestamp,
"interval":
1,
"partition":
1,
"offset":
1,
}
def send_update(self, rule, value, time_delta=None):
if time_delta is None:
time_delta = timedelta()
subscription = rule.query_subscription
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription,
value=value,
time_delta=time_delta)
processor.process_update(message)
return processor
def assert_no_active_incident(self, rule):
assert not self.active_incident_exists(rule)
def assert_active_incident(self, rule):
assert self.active_incident_exists(rule)
def active_incident_exists(self, rule):
return Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
status=IncidentStatus.OPEN.value,
alert_rule=rule,
).exists()
def assert_trigger_counts(self,
processor,
alert_triggers=0,
resolve_triggers=0):
assert processor.alert_triggers == alert_triggers
assert processor.resolve_triggers == resolve_triggers
assert get_alert_rule_stats(
processor.alert_rule)[1:] == (alert_triggers, resolve_triggers)
def test_removed_alert_rule(self):
message = self.build_subscription_update()
SubscriptionProcessor(self.subscription).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription")
# TODO: Check subscription is deleted once we start doing that
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.rule.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.rule.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update")
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.rule.alert_threshold,
timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update")
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.rule.alert_threshold,
timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold)
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(self.rule)
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1)
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
def test_alert_multiple_triggers(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-3))
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 1, 0)
self.assert_no_active_incident(rule)
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
processor = self.send_update(rule, rule.resolve_threshold - 1)
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_resolve_multiple_triggers(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-3))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
def test_resolve_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-4))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1,
timedelta(minutes=-3))
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 1)
self.assert_active_incident(rule)
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, rule.alert_threshold + 1,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
processor = self.send_update(rule, rule.alert_threshold - 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, rule.alert_threshold - 1,
timedelta(minutes=-3))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold - 1,
timedelta(minutes=-2))
self.assert_trigger_counts(processor, 0, 0)
self.assert_active_incident(rule)
processor = self.send_update(rule, rule.resolve_threshold + 1,
timedelta(minutes=-1))
self.assert_trigger_counts(processor, 0, 0)
self.assert_no_active_incident(rule)
class UpdateIncidentStatus(TestCase):
record_event = patcher('sentry.analytics.base.Analytics.record_event')
def get_most_recent_incident_activity(self, incident):
return IncidentActivity.objects.filter(
incident=incident).order_by('-id')[:1].get()
def test_status_already_set(self):
incident = self.create_incident(status=IncidentStatus.OPEN.value)
with self.assertRaises(StatusAlreadyChangedError):
update_incident_status(incident, IncidentStatus.OPEN)
def run_test(
self,
incident,
status,
expected_date_closed,
user=None,
comment=None,
):
prev_status = incident.status
self.record_event.reset_mock()
update_incident_status(incident, status, user=user, comment=comment)
incident = Incident.objects.get(id=incident.id)
assert incident.status == status.value
assert incident.date_closed == expected_date_closed
activity = self.get_most_recent_incident_activity(incident)
assert activity.type == IncidentActivityType.STATUS_CHANGE.value
assert activity.user == user
if user:
assert IncidentSubscription.objects.filter(incident=incident,
user=user).exists()
assert activity.value == six.text_type(status.value)
assert activity.previous_value == six.text_type(prev_status)
assert activity.comment == comment
assert activity.event_stats_snapshot is None
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentStatusUpdatedEvent)
assert event.data == {
'organization_id': six.text_type(self.organization.id),
'incident_id': six.text_type(incident.id),
'incident_type': six.text_type(incident.type),
'prev_status': six.text_type(prev_status),
'status': six.text_type(incident.status),
}
def test_closed(self):
incident = self.create_incident()
self.run_test(incident, IncidentStatus.CLOSED, timezone.now())
def test_reopened(self):
incident = self.create_incident(status=IncidentStatus.CLOSED.value,
date_closed=timezone.now())
self.run_test(incident, IncidentStatus.OPEN, None)
def test_all_params(self):
incident = self.create_incident()
self.run_test(
incident,
IncidentStatus.CLOSED,
timezone.now(),
user=self.user,
comment='lol',
)
class CreateIncidentActivityTest(TestCase, BaseIncidentsTest):
send_subscriber_notifications = patcher("sentry.incidents.tasks.send_subscriber_notifications")
record_event = patcher("sentry.analytics.base.Analytics.record_event")
def assert_notifications_sent(self, activity):
self.send_subscriber_notifications.apply_async.assert_called_once_with(
kwargs={"activity_id": activity.id}, countdown=10
)
def test_no_snapshot(self):
incident = self.create_incident()
self.record_event.reset_mock()
activity = create_incident_activity(
incident,
IncidentActivityType.STATUS_CHANGE,
user=self.user,
value=six.text_type(IncidentStatus.CLOSED.value),
previous_value=six.text_type(IncidentStatus.OPEN.value),
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.STATUS_CHANGE.value
assert activity.user == self.user
assert activity.value == six.text_type(IncidentStatus.CLOSED.value)
assert activity.previous_value == six.text_type(IncidentStatus.OPEN.value)
self.assert_notifications_sent(activity)
assert not self.record_event.called
def test_snapshot(self):
self.create_event(self.now - timedelta(minutes=2))
self.create_event(self.now - timedelta(minutes=2))
self.create_event(self.now - timedelta(minutes=1))
# Define events outside incident range. Should be included in the
# snapshot
self.create_event(self.now - timedelta(minutes=20))
self.create_event(self.now - timedelta(minutes=30))
# Too far out, should be excluded
self.create_event(self.now - timedelta(minutes=100))
incident = self.create_incident(
date_started=self.now - timedelta(minutes=5), query="", projects=[self.project]
)
event_stats_snapshot = create_initial_event_stats_snapshot(incident)
self.record_event.reset_mock()
activity = create_incident_activity(
incident, IncidentActivityType.CREATED, event_stats_snapshot=event_stats_snapshot
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.CREATED.value
assert activity.value is None
assert activity.previous_value is None
assert event_stats_snapshot == activity.event_stats_snapshot
self.assert_notifications_sent(activity)
assert not self.record_event.called
def test_comment(self):
incident = self.create_incident()
comment = "hello"
with self.assertChanges(
lambda: IncidentSubscription.objects.filter(incident=incident, user=self.user).exists(),
before=False,
after=True,
):
self.record_event.reset_mock()
activity = create_incident_activity(
incident, IncidentActivityType.COMMENT, user=self.user, comment=comment
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.COMMENT.value
assert activity.user == self.user
assert activity.comment == comment
assert activity.value is None
assert activity.previous_value is None
self.assert_notifications_sent(activity)
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentCommentCreatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(incident.type),
"user_id": six.text_type(self.user.id),
"activity_id": six.text_type(activity.id),
}
def test_mentioned_user_ids(self):
incident = self.create_incident()
mentioned_member = self.create_user()
subscribed_mentioned_member = self.create_user()
IncidentSubscription.objects.create(incident=incident, user=subscribed_mentioned_member)
comment = "hello **@%s** and **@%s**" % (
mentioned_member.username,
subscribed_mentioned_member.username,
)
with self.assertChanges(
lambda: IncidentSubscription.objects.filter(
incident=incident, user=mentioned_member
).exists(),
before=False,
after=True,
):
self.record_event.reset_mock()
activity = create_incident_activity(
incident,
IncidentActivityType.COMMENT,
user=self.user,
comment=comment,
mentioned_user_ids=[mentioned_member.id, subscribed_mentioned_member.id],
)
assert activity.incident == incident
assert activity.type == IncidentActivityType.COMMENT.value
assert activity.user == self.user
assert activity.comment == comment
assert activity.value is None
assert activity.previous_value is None
self.assert_notifications_sent(activity)
assert len(self.record_event.call_args_list) == 1
event = self.record_event.call_args[0][0]
assert isinstance(event, IncidentCommentCreatedEvent)
assert event.data == {
"organization_id": six.text_type(self.organization.id),
"incident_id": six.text_type(incident.id),
"incident_type": six.text_type(incident.type),
"user_id": six.text_type(self.user.id),
"activity_id": six.text_type(activity.id),
}
