def test_handled(self):
mock_handler = Mock()
type = AlertRuleTriggerAction.Type.EMAIL
AlertRuleTriggerAction.register_type("something", type, [])(mock_handler)
trigger = AlertRuleTriggerAction(type=AlertRuleTriggerAction.Type.EMAIL.value)
incident = Mock()
project = Mock()
trigger.build_handler(trigger, incident, project)
mock_handler.assert_called_once_with(trigger, incident, project)
assert not self.metrics.incr.called
def test_no_access(self):
result = access.DEFAULT
assert not result.is_active
assert result.sso_is_valid
assert not result.scopes
assert not result.has_team_access(Mock())
assert not result.has_team_scope(Mock(), "project:read")
assert not result.has_project_access(Mock())
assert not result.has_projects_access([Mock()])
assert not result.has_project_scope(Mock(), "project:read")
assert not result.has_project_membership(Mock())
def test_unignoring_group(self, send_robust):
group = self.create_group(status=GroupStatus.IGNORED)
request = self.make_request(user=self.user, method="GET")
request.user = self.user
request.data = {"status": "unresolved"}
request.GET = QueryDict(query_string=f"id={group.id}")
search_fn = Mock()
update_groups(
request, request.GET.getlist("id"), [self.project], self.organization.id, search_fn
)
group.refresh_from_db()
assert group.status == GroupStatus.UNRESOLVED
assert send_robust.called
def test_unresolving_resolved_group(self, send_robust, send_unresolved):
resolved_group = self.create_group(status=GroupStatus.RESOLVED)
assert resolved_group.status == GroupStatus.RESOLVED
request = self.make_request(user=self.user, method="GET")
request.user = self.user
request.data = {"status": "unresolved"}
request.GET = QueryDict(query_string=f"id={resolved_group.id}")
search_fn = Mock()
update_groups(request, [self.project], self.organization.id, search_fn)
resolved_group.refresh_from_db()
assert resolved_group.status == GroupStatus.UNRESOLVED
assert not send_robust.called
assert send_unresolved.called
def test_ignoring_group(self, send_robust):
group = self.create_group()
add_group_to_inbox(group, GroupInboxReason.NEW)
request = self.make_request(user=self.user, method="GET")
request.user = self.user
request.data = {"status": "ignored"}
request.GET = QueryDict(query_string="id={}".format(group.id))
search_fn = Mock()
update_groups(request, [self.project], self.organization.id, search_fn)
group.refresh_from_db()
assert group.status == GroupStatus.IGNORED
assert send_robust.called
assert not GroupInbox.objects.filter(group=group).exists()
def test_mark_reviewed_group(self, send_robust):
group = self.create_group()
add_group_to_inbox(group, GroupInboxReason.NEW)
request = self.make_request(user=self.user, method="GET")
request.user = self.user
request.data = {"inbox": False}
request.GET = QueryDict(query_string=f"id={group.id}")
search_fn = Mock()
update_groups(request, request.GET.getlist("id"), [self.project],
self.organization.id, search_fn)
group.refresh_from_db()
assert not GroupInbox.objects.filter(group=group).exists()
assert send_robust.called
def test_resolving_unresolved_group(self, send_robust):
unresolved_group = self.create_group(status=GroupStatus.UNRESOLVED)
add_group_to_inbox(unresolved_group, GroupInboxReason.NEW)
assert unresolved_group.status == GroupStatus.UNRESOLVED
request = self.make_request(user=self.user, method="GET")
request.user = self.user
request.data = {"status": "resolved"}
request.GET = QueryDict(query_string=f"id={unresolved_group.id}")
search_fn = Mock()
update_groups(request, [self.project], self.organization.id, search_fn)
unresolved_group.refresh_from_db()
assert unresolved_group.status == GroupStatus.RESOLVED
assert not GroupInbox.objects.filter(group=unresolved_group).exists()
assert send_robust.called
def test_resolving_unresolved_group(self, send_robust):
unresolved_group = self.create_group(status=GroupStatus.UNRESOLVED)
add_group_to_inbox(unresolved_group, GroupInboxReason.NEW)
assert unresolved_group.status == GroupStatus.UNRESOLVED
request = self.make_request(user=self.user, method="GET")
request.user = self.user
request.data = {"status": "resolved"}
request.GET = QueryDict(query_string="id={}".format(unresolved_group.id))
search_fn = Mock()
update_groups(request, [self.project], self.organization.id, search_fn)
unresolved_group.refresh_from_db()
assert unresolved_group.status == GroupStatus.RESOLVED
# TODO: Chris F.: This is temporarily removed while we perform some migrations.
# assert not GroupInbox.objects.filter(group=unresolved_group).exists()
assert send_robust.called
def test_no_accounts_received(self, mock_render_to_response, mock_get_user_info):
responses.reset()
responses.add(
responses.GET,
"https://app.vssps.visualstudio.com/_apis/accounts",
json={"value": [], "count": 0},
status=200,
)
view = AccountConfigView()
request = Mock()
request.POST = {}
request.user = self.user
pipeline = Mock()
pipeline.fetch_state = lambda key: {"data": {"access_token": "1234567890"}}
pipeline.organization = self.organization
view.dispatch(request, pipeline)
assert mock_get_user_info.called is True
assert mock_render_to_response.called is True
assert mock_render_to_response.call_args[1]["context"] == {"no_accounts": True}
def test_dispatch(self):
view = AccountConfigView()
request = HttpRequest()
request.POST = {"account": "1234567-8910"}
pipeline = Mock()
pipeline.state = {
"accounts": self.accounts,
"identity": {"data": {"access_token": "123456789"}},
}
pipeline.fetch_state = lambda key: pipeline.state[key]
pipeline.bind_state = lambda name, value: pipeline.state.update({name: value})
view.dispatch(request, pipeline)
assert pipeline.fetch_state(key="account") == self.accounts[1]
assert pipeline.next_step.call_count == 1
def test_middleware_as_superuser(self):
request = self.build_request()
delattr(request, "superuser")
delattr(request, "is_superuser")
middleware = SuperuserMiddleware()
middleware.process_request(request)
assert request.superuser.is_active
assert request.is_superuser()
response = Mock()
middleware.process_response(request, response)
response.set_signed_cookie.assert_called_once_with(
COOKIE_NAME,
request.superuser.token,
salt=COOKIE_SALT,
max_age=None,
secure=request.is_secure() if COOKIE_SECURE is None else COOKIE_SECURE,
httponly=COOKIE_HTTPONLY,
path=COOKIE_PATH,
domain=COOKIE_DOMAIN,
)
def test_basic_flow(self, mock_sha):
sha = Mock()
sha.hexdigest.return_value = "secret-token"
mock_sha.return_value = sha
self.assert_setup_flow()
integration = Integration.objects.get(provider=self.provider.key)
assert integration.external_id == "gitlab.example.com:4"
assert integration.name == "Cool"
assert integration.metadata == {
"instance": "gitlab.example.com",
"scopes": ["api"],
"icon":
"https://gitlab.example.com/uploads/group/avatar/4/foo.jpg",
"domain_name": "gitlab.example.com/cool-group",
"verify_ssl": True,
"base_url": "https://gitlab.example.com",
"webhook_secret": "secret-token",
"group_id": self.default_group_id,
"include_subgroups": True,
}
oi = OrganizationIntegration.objects.get(
integration=integration, organization=self.organization)
assert oi.config == {}
idp = IdentityProvider.objects.get(type="gitlab")
identity = Identity.objects.get(
idp=idp,
user=self.user,
external_id="gitlab.example.com:user_id_1")
assert identity.status == IdentityStatus.VALID
assert identity.data == {
"access_token": "xxxxx-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx"
}
def setUp(self):
self.event = Mock()
self.event.data = {}
class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
def setUp(self):
super(ProcessUpdateTest, self).setUp()
self.old_handlers = AlertRuleTriggerAction._type_registrations
AlertRuleTriggerAction._type_registrations = {}
self.email_action_handler = Mock()
AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])(
self.email_action_handler
)
self._run_tasks = self.tasks()
self._run_tasks.__enter__()
def tearDown(self):
super(ProcessUpdateTest, self).tearDown()
AlertRuleTriggerAction._type_registrations = self.old_handlers
self._run_tasks.__exit__(None, None, None)
@fixture
def other_project(self):
return self.create_project()
@fixture
def sub(self):
return self.rule.query_subscriptions.filter(project=self.project).get()
@fixture
def other_sub(self):
return self.rule.query_subscriptions.filter(project=self.other_project).get()
@fixture
def rule(self):
rule = create_alert_rule(
self.organization,
[self.project, self.other_project],
"some rule",
query="",
aggregation=QueryAggregations.TOTAL,
time_window=1,
threshold_period=1,
)
# Make sure the trigger exists
trigger = create_alert_rule_trigger(
rule, "hi", AlertRuleThresholdType.ABOVE, 100, resolve_threshold=10
)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
six.text_type(self.user.id),
)
return rule
@fixture
def trigger(self):
return self.rule.alertruletrigger_set.get()
@fixture
def action(self):
return self.trigger.alertruletriggeraction_set.get()
def build_subscription_update(self, subscription, time_delta=None, value=None):
if time_delta is not None:
timestamp = timezone.now() + time_delta
else:
timestamp = timezone.now()
timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0)
data = {}
if subscription:
aggregation_type = query_aggregation_to_snuba[
QueryAggregations(subscription.aggregation)
]
value = randint(0, 100) if value is None else value
data = {aggregation_type[2]: value}
values = {"data": [data]}
return {
"subscription_id": subscription.subscription_id if subscription else uuid4().hex,
"values": values,
"timestamp": timestamp,
"interval": 1,
"partition": 1,
"offset": 1,
}
def send_update(self, rule, value, time_delta=None, subscription=None):
self.email_action_handler.reset_mock()
if time_delta is None:
time_delta = timedelta()
if subscription is None:
subscription = self.sub
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription, value=value, time_delta=time_delta)
processor.process_update(message)
return processor
def assert_trigger_exists_with_status(self, incident, trigger, status):
assert IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger, status=status.value
).exists()
def assert_trigger_does_not_exist_for_incident(self, incident, trigger):
assert not IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger
).exists()
def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None):
if incidents_to_exclude is None:
incidents_to_exclude = []
assert (
not IncidentTrigger.objects.filter(alert_rule_trigger=trigger)
.exclude(incident__in=incidents_to_exclude)
.exists()
)
def assert_action_handler_called_with_actions(self, incident, actions, project=None):
project = self.project if project is None else project
if not actions:
if not incident:
assert not self.email_action_handler.called
else:
for call_args in self.email_action_handler.call_args_list:
assert call_args[0][1] != incident
else:
self.email_action_handler.assert_has_calls(
[call(action, incident, project) for action in actions], any_order=True
)
def assert_actions_fired_for_incident(self, incident, actions=None, project=None):
actions = [] if actions is None else actions
project = self.project if project is None else project
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list)
def assert_actions_resolved_for_incident(self, incident, actions=None, project=None):
project = self.project if project is None else project
actions = [] if actions is None else actions
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list)
def assert_no_active_incident(self, rule, subscription=None):
assert not self.active_incident_exists(rule, subscription=subscription)
def assert_active_incident(self, rule, subscription=None):
incidents = self.active_incident_exists(rule, subscription=subscription)
assert incidents
return incidents[0]
def active_incident_exists(self, rule, subscription=None):
if subscription is None:
subscription = self.sub
return list(
Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
alert_rule=rule,
projects=subscription.project,
).exclude(status=IncidentStatus.CLOSED.value)
)
def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0):
assert processor.trigger_alert_counts[trigger.id] == alert_triggers
assert processor.trigger_resolve_counts[trigger.id] == resolve_triggers
alert_stats, resolve_stats = get_alert_rule_stats(
processor.alert_rule, processor.subscription, [trigger]
)[1:]
assert alert_stats[trigger.id] == alert_triggers
assert resolve_stats[trigger.id] == resolve_triggers
def test_removed_alert_rule(self):
message = self.build_subscription_update(self.sub)
self.rule.delete()
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription"
)
# TODO: Check subscription is deleted once we start doing that
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(self.rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.resolve_threshold - 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
trigger = self.trigger
trigger.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
trigger = self.trigger
trigger.update(threshold_type=AlertRuleThresholdType.BELOW.value)
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, trigger.resolve_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_subscriptions_do_not_conflict(self):
# Verify that multiple subscriptions associated with a rule don't conflict with
# each other
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
# Send an update through for the first subscription. This shouldn't trigger an
# incident, since we need two consecutive updates that are over the threshold.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Have an update come through for the other sub. This shouldn't influence the original
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Send another update through for the first subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger, [incident])
# Send another update through for the second subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project)
# Now we want to test that resolving is isolated. Send another update through
# for the first subscription.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
# This second update for the second subscription should resolve its incident,
# but not the incident from the first subscription.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project)
# This second update for the first subscription should resolve its incident now.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(other_incident, [])
def test_multiple_triggers(self):
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 1, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(trigger)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_action_handler_called_with_actions(None, [])
# This should cause both to increment, although only `trigger` should fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 1, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_actions_fired_for_incident(incident, [self.action])
# Now only `other_trigger` should increment and fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [other_action])
# Now send through two updates where we're below threshold for `other_trigger`.
# The trigger should end up resolved, but the incident should still be active
processor = self.send_update(
rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, other_trigger.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [other_action])
# Now we push the other trigger below the resolve threshold twice. This should
# close the incident.
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-5), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 1)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-4), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_triggers_at_same_time(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200, resolve_threshold=50
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_one_with_no_resolve(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(
self.rule, "hello", AlertRuleThresholdType.ABOVE, 200
)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, trigger.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_resolved_for_incident(incident, [self.action])
def build_mock_message(self, data, topic=None):
message = Mock()
message.value.return_value = json.dumps(data)
if topic:
message.topic.return_value = topic
return message
class DiscoverLinkSharedEvent(BaseEventTest):
@responses.activate
@patch(
"sentry.integrations.slack.endpoints.event.match_link",
# match_link will be called twice, for each our links. Resolve into
# two unique links and one duplicate.
side_effect=[
(LinkType.DISCOVER, {"arg1": "value1"}),
(LinkType.DISCOVER, {"arg1", "value2"}),
(LinkType.DISCOVER, {"arg1": "value1"}),
],
)
@patch("sentry.integrations.slack.requests.event.has_discover_links", return_value=True)
@patch(
"sentry.integrations.slack.endpoints.event.link_handlers",
{
LinkType.DISCOVER: Handler(
matcher=re.compile(r"test"),
arg_mapper=make_type_coercer({}),
fn=Mock(return_value={"link1": "unfurl", "link2": "unfurl"}),
)
},
)
def share_discover_links(self, mock_match_link, mock_):
responses.add(responses.POST, "https://slack.com/api/chat.postEphemeral", json={"ok": True})
responses.add(responses.POST, "https://slack.com/api/chat.unfurl", json={"ok": True})
resp = self.post_webhook(event_data=json.loads(LINK_SHARED_EVENT))
assert resp.status_code == 200, resp.content
data = dict(parse_qsl(responses.calls[0].request.body))
return data
def test_share_discover_links_unlinked_user(self):
IdentityProvider.objects.create(type="slack", external_id="TXXXXXXX1", config={})
with self.feature("organizations:chart-unfurls"):
data = self.share_discover_links()
blocks = json.loads(data["blocks"])
assert blocks[0]["type"] == "section"
assert (
blocks[0]["text"]["text"]
== "Link your Slack identity to Sentry to unfurl Discover charts."
)
assert blocks[1]["type"] == "actions"
assert len(blocks[1]["elements"]) == 2
assert [button["text"]["text"] for button in blocks[1]["elements"]] == ["Link", "Cancel"]
def test_share_discover_links_linked_user(self):
idp = IdentityProvider.objects.create(type="slack", external_id="TXXXXXXX1", config={})
Identity.objects.create(
external_id="Uxxxxxxx",
idp=idp,
user=self.user,
status=IdentityStatus.VALID,
scopes=[],
)
data = self.share_discover_links()
unfurls = json.loads(data["unfurls"])
# We only have two unfurls since one link was duplicated
assert len(unfurls) == 2
assert unfurls["link1"] == "unfurl"
assert unfurls["link2"] == "unfurl"
def test_shutdown(self):
self.producer.produce(self.topic, json.dumps(self.valid_wrapper))
valid_wrapper_2 = deepcopy(self.valid_wrapper)
valid_wrapper_2["payload"]["result"]["hello"] = 25
valid_wrapper_3 = deepcopy(valid_wrapper_2)
valid_wrapper_3["payload"]["result"]["hello"] = 5000
self.producer.produce(self.topic, json.dumps(valid_wrapper_2))
self.producer.flush()
counts = [0]
def mock_callback(*args, **kwargs):
counts[0] += 1
if counts[0] > 1:
raise KeyboardInterrupt()
mock = Mock()
mock.side_effect = mock_callback
register_subscriber(self.registration_key)(mock)
sub = QuerySubscription.objects.create(
project=self.project,
type=self.registration_key,
subscription_id=self.subscription_id,
dataset="something",
query="hello",
aggregation=0,
time_window=1,
resolution=1,
)
consumer = QuerySubscriptionConsumer("hi",
topic=self.topic,
commit_batch_size=100)
consumer.run()
valid_payload = self.valid_payload
valid_payload["values"] = valid_payload["result"]
valid_payload["timestamp"] = parse_date(
valid_payload["timestamp"]).replace(tzinfo=pytz.utc)
valid_wrapper_2["payload"]["values"] = valid_wrapper_2["payload"][
"result"]
valid_wrapper_2["payload"]["timestamp"] = parse_date(
valid_wrapper_2["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls(
[call(valid_payload, sub),
call(valid_wrapper_2["payload"], sub)])
# Offset should be committed for the first message, so second run should process
# the second message again
self.producer.produce(self.topic, json.dumps(valid_wrapper_3))
self.producer.flush()
mock.reset_mock()
counts[0] = 0
consumer.run()
valid_wrapper_3["payload"]["values"] = valid_wrapper_3["payload"][
"result"]
valid_wrapper_3["payload"]["timestamp"] = parse_date(
valid_wrapper_3["payload"]["timestamp"]).replace(tzinfo=pytz.utc)
mock.assert_has_calls([
call(valid_wrapper_2["payload"], sub),
call(valid_wrapper_3["payload"], sub)
])
def test_no_handler(self):
trigger = AlertRuleTriggerAction(
type=AlertRuleTriggerAction.Type.EMAIL.value)
assert trigger.fire(Mock(), Mock(), 123) is None
def test_unhandled(self):
trigger = AlertRuleTriggerAction(
type=AlertRuleTriggerAction.Type.EMAIL.value)
trigger.build_handler(Mock(), Mock())
self.metrics.incr.assert_called_once_with(
"alert_rule_trigger.unhandled_type.0")
def build_mock_message(self, data):
message = Mock()
message.value.return_value = json.dumps(data)
return message
def build_mock(**attrs):
obj = Mock()
for key, value in attrs.items():
setattr(obj, key, value)
obj.__repr__ = lambda x: repr(attrs)
return obj
def setUp(self):
super(SnubaEventStreamTest, self).setUp()
self.kafka_eventstream = KafkaEventStream()
self.kafka_eventstream.producer = Mock()
class MailPluginTest(TestCase):
@fixture
def plugin(self):
return MailPlugin()
@mock.patch("sentry.models.ProjectOption.objects.get_value",
Mock(side_effect=lambda p, k, d, **kw: d))
@mock.patch(
"sentry.plugins.sentry_mail.models.MailPlugin.get_sendable_users",
Mock(return_value=[]))
def test_should_notify_no_sendable_users(self):
assert not self.plugin.should_notify(group=Mock(), event=Mock())
def test_simple_notification(self):
event = self.store_event(data={
"message": "Hello world",
"level": "error"
},
project_id=self.project.id)
rule = Rule.objects.create(project=self.project, label="my rule")
notification = Notification(event=event, rule=rule)
with self.options({"system.url-prefix":
"http://example.com"}), self.tasks():
self.plugin.notify(notification)
msg = mail.outbox[0]
assert msg.subject == "[Sentry] BAR-1 - Hello world"
assert "my rule" in msg.alternatives[0][0]
@mock.patch("sentry.interfaces.stacktrace.Stacktrace.get_title")
@mock.patch("sentry.interfaces.stacktrace.Stacktrace.to_email_html")
@mock.patch("sentry.plugins.sentry_mail.models.MailPlugin._send_mail")
def test_notify_users_renders_interfaces_with_utf8(self, _send_mail,
_to_email_html,
_get_title):
_to_email_html.return_value = u"רונית מגן"
_get_title.return_value = "Stacktrace"
event = self.store_event(
data={
"message": "Soubor ji\xc5\xbe existuje",
"stacktrace": {
"frames": [{}]
}
},
project_id=self.project.id,
)
notification = Notification(event=event)
with self.options({"system.url-prefix": "http://example.com"}):
self.plugin.notify(notification)
_get_title.assert_called_once_with()
_to_email_html.assert_called_once_with(event)
@mock.patch("sentry.plugins.sentry_mail.models.MailPlugin._send_mail")
def test_notify_users_does_email(self, _send_mail):
event_manager = EventManager({
"message": "hello world",
"level": "error"
})
event_manager.normalize()
event_data = event_manager.get_data()
event_type = event_manager.get_event_type()
event_data["type"] = event_type.key
event_data["metadata"] = event_type.get_metadata(event_data)
event = event_manager.save(self.project.id)
group = event.group
notification = Notification(event=event)
with self.options({"system.url-prefix": "http://example.com"}):
self.plugin.notify(notification)
assert _send_mail.call_count == 1
args, kwargs = _send_mail.call_args
self.assertEquals(kwargs.get("project"), self.project)
self.assertEquals(kwargs.get("reference"), group)
assert kwargs.get("subject") == u"BAR-1 - hello world"
@mock.patch("sentry.plugins.sentry_mail.models.MailPlugin._send_mail")
def test_multiline_error(self, _send_mail):
event_manager = EventManager({
"message": "hello world\nfoo bar",
"level": "error"
})
event_manager.normalize()
event_data = event_manager.get_data()
event_type = event_manager.get_event_type()
event_data["type"] = event_type.key
event_data["metadata"] = event_type.get_metadata(event_data)
event = event_manager.save(self.project.id)
notification = Notification(event=event)
with self.options({"system.url-prefix": "http://example.com"}):
self.plugin.notify(notification)
assert _send_mail.call_count == 1
args, kwargs = _send_mail.call_args
assert kwargs.get("subject") == u"BAR-1 - hello world"
def test_get_sendable_users(self):
from sentry.models import UserOption, User
user = self.create_user(email="*****@*****.**", is_active=True)
user2 = self.create_user(email="*****@*****.**", is_active=True)
self.create_user(email="*****@*****.**", is_active=True)
# user with inactive account
self.create_user(email="*****@*****.**", is_active=False)
# user not in any groups
self.create_user(email="*****@*****.**", is_active=True)
organization = self.create_organization(owner=user)
team = self.create_team(organization=organization)
project = self.create_project(name="Test", teams=[team])
OrganizationMemberTeam.objects.create(
organizationmember=OrganizationMember.objects.get(
user=user, organization=organization),
team=team,
)
self.create_member(user=user2, organization=organization, teams=[team])
# all members
assert sorted(set([user.pk, user2.pk])) == sorted(
self.plugin.get_sendable_users(project))
# disabled user2
UserOption.objects.create(key="mail:alert",
value=0,
project=project,
user=user2)
assert user2.pk not in self.plugin.get_sendable_users(project)
user4 = User.objects.create(username="******",
email="*****@*****.**",
is_active=True)
self.create_member(user=user4, organization=organization, teams=[team])
assert user4.pk in self.plugin.get_sendable_users(project)
# disabled by default user4
uo1 = UserOption.objects.create(key="subscribe_by_default",
value="0",
project=project,
user=user4)
assert user4.pk not in self.plugin.get_sendable_users(project)
uo1.delete()
UserOption.objects.create(key="subscribe_by_default",
value=u"0",
project=project,
user=user4)
assert user4.pk not in self.plugin.get_sendable_users(project)
def test_notify_users_with_utf8_subject(self):
event = self.store_event(data={
"message": "רונית מגן",
"level": "error"
},
project_id=self.project.id)
notification = Notification(event=event)
with self.options({"system.url-prefix":
"http://example.com"}), self.tasks():
self.plugin.notify(notification)
assert len(mail.outbox) == 1
msg = mail.outbox[0]
assert msg.subject == u"[Sentry] BAR-1 - רונית מגן"
def test_get_digest_subject(self):
assert (self.plugin.get_digest_subject(
mock.Mock(qualified_short_id="BAR-1"),
{mock.sentinel.group: 3},
datetime(2016, 9, 19, 1, 2, 3, tzinfo=pytz.utc),
) == "BAR-1 - 1 new alert since Sept. 19, 2016, 1:02 a.m. UTC")
@mock.patch.object(MailPlugin,
"notify",
side_effect=MailPlugin.notify,
autospec=True)
def test_notify_digest(self, notify):
project = self.project
event = self.store_event(
data={
"timestamp": iso_format(before_now(minutes=1)),
"fingerprint": ["group-1"]
},
project_id=project.id,
)
event2 = self.store_event(
data={
"timestamp": iso_format(before_now(minutes=1)),
"fingerprint": ["group-2"]
},
project_id=project.id,
)
rule = project.rule_set.all()[0]
digest = build_digest(
project,
(event_to_record(event,
(rule, )), event_to_record(event2, (rule, ))))
with self.tasks():
self.plugin.notify_digest(project, digest)
assert notify.call_count == 0
assert len(mail.outbox) == 1
message = mail.outbox[0]
assert "List-ID" in message.message()
@mock.patch.object(MailPlugin,
"notify",
side_effect=MailPlugin.notify,
autospec=True)
@mock.patch.object(MessageBuilder, "send_async", autospec=True)
def test_notify_digest_single_record(self, send_async, notify):
event = self.store_event(data={}, project_id=self.project.id)
rule = self.project.rule_set.all()[0]
digest = build_digest(self.project, (event_to_record(event,
(rule, )), ))
self.plugin.notify_digest(self.project, digest)
assert send_async.call_count == 1
assert notify.call_count == 1
def test_notify_digest_subject_prefix(self):
ProjectOption.objects.set_value(project=self.project,
key=u"mail:subject_prefix",
value="[Example prefix] ")
event = self.store_event(
data={
"timestamp": iso_format(before_now(minutes=1)),
"fingerprint": ["group-1"]
},
project_id=self.project.id,
)
event2 = self.store_event(
data={
"timestamp": iso_format(before_now(minutes=1)),
"fingerprint": ["group-2"]
},
project_id=self.project.id,
)
rule = self.project.rule_set.all()[0]
digest = build_digest(
self.project,
(event_to_record(event,
(rule, )), event_to_record(event2, (rule, ))))
with self.tasks():
self.plugin.notify_digest(self.project, digest)
assert len(mail.outbox) == 1
msg = mail.outbox[0]
assert msg.subject.startswith("[Example prefix]")
def test_assignment(self):
UserOption.objects.set_value(user=self.user,
key="workflow:notifications",
value=UserOptionValue.all_conversations)
activity = Activity.objects.create(
project=self.project,
group=self.group,
type=Activity.ASSIGNED,
user=self.create_user("*****@*****.**"),
data={
"assignee": six.text_type(self.user.id),
"assigneeType": "user"
},
)
with self.tasks():
self.plugin.notify_about_activity(activity)
assert len(mail.outbox) == 1
msg = mail.outbox[0]
assert (
msg.subject ==
"Re: [Sentry] BAR-1 - \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf"
)
assert msg.to == [self.user.email]
def test_assignment_team(self):
UserOption.objects.set_value(user=self.user,
key="workflow:notifications",
value=UserOptionValue.all_conversations)
activity = Activity.objects.create(
project=self.project,
group=self.group,
type=Activity.ASSIGNED,
user=self.create_user("*****@*****.**"),
data={
"assignee": six.text_type(self.project.teams.first().id),
"assigneeType": "team"
},
)
with self.tasks():
self.plugin.notify_about_activity(activity)
assert len(mail.outbox) == 1
msg = mail.outbox[0]
assert (
msg.subject ==
"Re: [Sentry] BAR-1 - \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf"
)
assert msg.to == [self.user.email]
def test_note(self):
user_foo = self.create_user("*****@*****.**")
UserOption.objects.set_value(user=self.user,
key="workflow:notifications",
value=UserOptionValue.all_conversations)
activity = Activity.objects.create(
project=self.project,
group=self.group,
type=Activity.NOTE,
user=user_foo,
data={"text": "sup guise"},
)
self.project.teams.first().organization.member_set.create(
user=user_foo)
with self.tasks():
self.plugin.notify_about_activity(activity)
assert len(mail.outbox) >= 1
msg = mail.outbox[-1]
assert (
msg.subject ==
"Re: [Sentry] BAR-1 - \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf"
)
assert msg.to == [self.user.email]
def test_notify_with_suspect_commits(self):
repo = Repository.objects.create(organization_id=self.organization.id,
name=self.organization.id)
release = self.create_release(project=self.project, version="v12")
release.set_commits([{
"id":
"a" * 40,
"repository":
repo.name,
"author_email":
"*****@*****.**",
"author_name":
"Bob",
"message":
"i fixed a bug",
"patch_set": [{
"path": "src/sentry/models/release.py",
"type": "M"
}],
}])
event = self.store_event(
data={
"message": "Kaboom!",
"platform": "python",
"timestamp": iso_format(before_now(seconds=1)),
"stacktrace": {
"frames": [
{
"function": "handle_set_commits",
"abs_path": "/usr/src/sentry/src/sentry/tasks.py",
"module": "sentry.tasks",
"in_app": True,
"lineno": 30,
"filename": "sentry/tasks.py",
},
{
"function": "set_commits",
"abs_path":
"/usr/src/sentry/src/sentry/models/release.py",
"module": "sentry.models.release",
"in_app": True,
"lineno": 39,
"filename": "sentry/models/release.py",
},
]
},
"tags": {
"sentry:release": release.version
},
},
project_id=self.project.id,
)
with self.tasks():
notification = Notification(event=event)
self.plugin.notify(notification)
assert len(mail.outbox) >= 1
msg = mail.outbox[-1]
assert "Suspect Commits" in msg.body
def test_should_notify_no_sendable_users(self):
assert not self.plugin.should_notify(group=Mock(), event=Mock())
def test_should_notify_sendable_users_not_has_issue_alerts_targetting(
self):
self.group.project.flags.has_issue_alerts_targeting = False
self.group.project.save()
assert self.plugin.should_notify(group=self.group, event=Mock())
class LinkSharedEventTest(BaseEventTest):
@responses.activate
@patch(
"sentry.integrations.slack.endpoints.event.match_link",
# match_link will be called twice, for each our links. Resolve into
# two unique links and one duplicate.
side_effect=[
("mock_link", {
"arg1": "value1"
}),
("mock_link", {"arg1", "value2"}),
("mock_link", {
"arg1": "value1"
}),
],
)
@patch(
"sentry.integrations.slack.endpoints.event.link_handlers",
{
"mock_link":
Handler(
matcher=re.compile(r"test"),
arg_mapper=make_type_coercer({}),
fn=Mock(return_value={
"link1": "unfurl",
"link2": "unfurl"
}),
)
},
)
def share_links(self, mock_match_link):
responses.add(responses.POST,
"https://slack.com/api/chat.unfurl",
json={"ok": True})
resp = self.post_webhook(event_data=json.loads(LINK_SHARED_EVENT))
assert resp.status_code == 200, resp.content
assert len(mock_match_link.mock_calls) == 3
data = dict(parse_qsl(responses.calls[0].request.body))
unfurls = json.loads(data["unfurls"])
# We only have two unfurls since one link was duplicated
assert len(unfurls) == 2
assert unfurls["link1"] == "unfurl"
assert unfurls["link2"] == "unfurl"
return data
def test_valid_token(self):
data = self.share_links()
assert data["token"] == "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx"
def test_user_access_token(self):
# this test is needed to make sure that classic bots installed by on-prem users
# still work since they needed to use a user_access_token for unfurl
self.integration.metadata.update({
"user_access_token":
"xoxt-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
"access_token":
"xoxm-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
})
self.integration.save()
data = self.share_links()
assert data["token"] == "xoxt-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx"
class StoreViewTest(TestCase):
@fixture
def path(self):
return reverse("sentry-api-store",
kwargs={"project_id": self.project.id})
@mock.patch("sentry.web.api.StoreView._parse_header")
def test_options_response(self, parse_header):
project = self.create_project()
pk = ProjectKey.objects.get_or_create(project=project)[0]
parse_header.return_value = {
"sentry_project": project.id,
"sentry_key": pk.public_key,
"sentry_version": "2.0",
}
resp = self.client.options(self.path)
assert resp.status_code == 200, (resp.status_code, resp.content)
assert resp.has_header("Allow")
self.assertEquals(resp["Allow"], "GET, POST, HEAD, OPTIONS")
assert resp.has_header("Content-Length")
self.assertEquals(resp["Content-Length"], "0")
def test_options_with_no_origin_or_referrer(self):
resp = self.client.options(self.path)
assert resp.status_code == 200, (resp.status_code, resp.content)
assert resp.has_header("Access-Control-Allow-Origin")
self.assertEquals(resp["Access-Control-Allow-Origin"], "*")
def test_options_response_with_valid_origin(self):
resp = self.client.options(self.path, HTTP_ORIGIN="http://foo.com")
assert resp.status_code == 200, (resp.status_code, resp.content)
assert resp.has_header("Access-Control-Allow-Origin")
self.assertEquals(resp["Access-Control-Allow-Origin"],
"http://foo.com")
def test_options_response_with_valid_referrer(self):
resp = self.client.options(self.path, HTTP_REFERER="http://foo.com")
assert resp.status_code == 200, (resp.status_code, resp.content)
assert resp.has_header("Access-Control-Allow-Origin")
self.assertEquals(resp["Access-Control-Allow-Origin"],
"http://foo.com")
def test_options_response_origin_preferred_over_referrer(self):
resp = self.client.options(self.path,
HTTP_REFERER="http://foo.com",
HTTP_ORIGIN="http://bar.com")
assert resp.status_code == 200, (resp.status_code, resp.content)
assert resp.has_header("Access-Control-Allow-Origin")
self.assertEquals(resp["Access-Control-Allow-Origin"],
"http://bar.com")
@mock.patch("sentry.event_manager.is_valid_ip",
mock.Mock(return_value=False))
def test_request_with_blacklisted_ip(self):
resp = self._postWithHeader({})
assert resp.status_code == 403, (resp.status_code, resp.content)
@mock.patch("sentry.event_manager.is_valid_release",
mock.Mock(return_value=False))
def test_request_with_filtered_release(self):
body = {
"release": "abcdefg",
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
@mock.patch("sentry.event_manager.is_valid_error_message",
mock.Mock(return_value=False))
def test_request_with_filtered_error(self):
body = {
"release": "abcdefg",
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
def test_request_with_invalid_ip(self):
self.project.update_option("sentry:blacklisted_ips", ["127.0.0.1"])
body = {
"release": "abcdefg",
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
def test_request_with_invalid_release(self):
self.project.update_option(u"sentry:{}".format(FilterTypes.RELEASES),
["1.3.2"])
body = {
"release": "1.3.2",
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
def test_request_with_short_release_globbing(self):
self.project.update_option(u"sentry:{}".format(FilterTypes.RELEASES),
["1.*"])
body = {
"release": "1.3.2",
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
def test_request_with_longer_release_globbing(self):
self.project.update_option(u"sentry:{}".format(FilterTypes.RELEASES),
["2.1.*"])
body = {
"release": "2.1.3",
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
def test_request_with_invalid_error_messages(self):
self.project.update_option(
u"sentry:{}".format(FilterTypes.ERROR_MESSAGES),
["ZeroDivisionError*"])
body = {
"release": "abcdefg",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
"logentry": {
"formatted":
"ZeroDivisionError: integer division or modulo by zero",
"message": "%s: integer division or modulo by zero",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
@mock.patch("sentry.relay.config.get_origins")
def test_request_with_bad_origin(self, get_origins):
get_origins.return_value = ["foo.com"]
body = {"logentry": {"formatted": "hello world"}}
resp = self._postWithHeader(body, HTTP_ORIGIN="lolnope.com")
assert resp.status_code == 403, (resp.status_code, resp.content)
assert b"Invalid origin" in resp.content
def test_request_with_beginning_glob(self):
self.project.update_option(
u"sentry:{}".format(FilterTypes.ERROR_MESSAGES),
["*: integer division or modulo by zero"],
)
body = {
"release": "abcdefg",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
"logentry": {
"message":
"ZeroDivisionError: integer division or modulo by zero",
"formatted": "",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 403, (resp.status_code, resp.content)
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrubs_ip_address(self, mock_insert_data_to_database):
self.project.update_option("sentry:scrub_ip_address", True)
body = {
"message": "foo bar",
"sdk": {
"name": "sentry-browser",
"version": "3.23.3",
"client_ip": "127.0.0.1"
},
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert not call_data["user"].get("ip_address")
assert not call_data["request"]["env"].get("REMOTE_ADDR")
assert not call_data["sdk"].get("client_ip")
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrubs_org_ip_address_override(self,
mock_insert_data_to_database):
self.organization.update_option("sentry:require_scrub_ip_address",
True)
self.project.update_option("sentry:scrub_ip_address", False)
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"env": {
"REMOTE_ADDR": "127.0.0.1"
},
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert not call_data["user"].get("ip_address")
assert not call_data["request"]["env"].get("REMOTE_ADDR")
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrub_data_off(self, mock_insert_data_to_database):
self.project.update_option("sentry:scrub_data", False)
self.project.update_option("sentry:scrub_defaults", False)
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"data": "password=lol&foo=1&bar=2&baz=3",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["request"]["data"] == {
"password": "******",
"foo": "1",
"bar": "2",
"baz": "3",
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrub_data_on(self, mock_insert_data_to_database):
self.project.update_option("sentry:scrub_data", True)
self.project.update_option("sentry:scrub_defaults", False)
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"data": "password=lol&foo=1&bar=2&baz=3",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["request"]["data"] == {
"password": "******",
"foo": "1",
"bar": "2",
"baz": "3",
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrub_data_defaults(self, mock_insert_data_to_database):
self.project.update_option("sentry:scrub_data", True)
self.project.update_option("sentry:scrub_defaults", True)
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"data": "password=lol&foo=1&bar=2&baz=3",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["request"]["data"] == {
"password": "******",
"foo": "1",
"bar": "2",
"baz": "3",
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrub_data_sensitive_fields(self, mock_insert_data_to_database):
self.project.update_option("sentry:scrub_data", True)
self.project.update_option("sentry:scrub_defaults", True)
self.project.update_option("sentry:sensitive_fields", ["foo", "bar"])
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"data": "password=lol&foo=1&bar=2&baz=3",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["request"]["data"] == {
"password": "******",
"foo": "[Filtered]",
"bar": "[Filtered]",
"baz": "3",
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrub_data_org_override(self, mock_insert_data_to_database):
self.organization.update_option("sentry:require_scrub_data", True)
self.project.update_option("sentry:scrub_data", False)
self.organization.update_option("sentry:require_scrub_defaults", True)
self.project.update_option("sentry:scrub_defaults", False)
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"data": "password=lol&foo=1&bar=2&baz=3",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["request"]["data"] == {
"password": "******",
"foo": "1",
"bar": "2",
"baz": "3",
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_scrub_data_org_override_sensitive_fields(
self, mock_insert_data_to_database):
self.organization.update_option("sentry:require_scrub_data", True)
self.organization.update_option("sentry:require_scrub_defaults", True)
self.organization.update_option("sentry:sensitive_fields", ["baz"])
self.project.update_option("sentry:sensitive_fields", ["foo", "bar"])
body = {
"message": "foo bar",
"user": {
"ip_address": "127.0.0.1"
},
"request": {
"method": "GET",
"url": "http://example.com/",
"data": "password=lol&foo=1&bar=2&baz=3",
},
}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["request"]["data"] == {
"password": "******",
"foo": "[Filtered]",
"bar": "[Filtered]",
"baz": "[Filtered]",
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database")
def test_uses_client_as_sdk(self, mock_insert_data_to_database):
body = {"message": "foo bar"}
resp = self._postWithHeader(body)
assert resp.status_code == 200, (resp.status_code, resp.content)
call_data = mock_insert_data_to_database.call_args[0][0]
assert call_data["sdk"] == {
"name": "_postWithHeader",
"version": "0.0.0"
}
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database",
Mock())
def test_accepted_signal(self):
mock_event_accepted = Mock()
event_accepted.connect(mock_event_accepted)
resp = self._postWithHeader({"logentry": {"message": u"hello"}})
assert resp.status_code == 200, resp.content
assert_mock_called_once_with_partial(mock_event_accepted,
ip="127.0.0.1",
project=self.project,
signal=event_accepted)
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database",
Mock())
@mock.patch("sentry.app.quotas.is_rate_limited")
def test_dropped_signal(self, mock_is_rate_limited):
mock_is_rate_limited.is_limited = True
mock_event_dropped = Mock()
event_dropped.connect(mock_event_dropped)
resp = self._postWithHeader({"logentry": {"message": u"hello"}})
assert resp.status_code == 429, resp.content
assert_mock_called_once_with_partial(mock_event_dropped,
ip="127.0.0.1",
project=self.project,
signal=event_dropped)
@mock.patch("sentry.coreapi.ClientApiHelper.insert_data_to_database",
Mock())
@mock.patch("sentry.event_manager.EventManager.should_filter")
def test_filtered_signal(self, mock_should_filter):
mock_should_filter.return_value = (True, "ip-address")
mock_event_filtered = Mock()
event_filtered.connect(mock_event_filtered)
resp = self._postWithHeader({"logentry": {"message": u"hello"}})
assert resp.status_code == 403, resp.content
assert_mock_called_once_with_partial(mock_event_filtered,
ip="127.0.0.1",
project=self.project,
signal=event_filtered)
class ProcessUpdateTest(TestCase):
metrics = patcher("sentry.incidents.subscription_processor.metrics")
slack_client = patcher("sentry.integrations.slack.utils.SlackClient.post")
def setUp(self):
super().setUp()
self.old_handlers = AlertRuleTriggerAction._type_registrations
AlertRuleTriggerAction._type_registrations = {}
self.email_action_handler = Mock()
AlertRuleTriggerAction.register_type("email", AlertRuleTriggerAction.Type.EMAIL, [])(
self.email_action_handler
)
self._run_tasks = self.tasks()
self._run_tasks.__enter__()
def tearDown(self):
super().tearDown()
AlertRuleTriggerAction._type_registrations = self.old_handlers
self._run_tasks.__exit__(None, None, None)
@fixture
def other_project(self):
return self.create_project()
@fixture
def sub(self):
return self.rule.snuba_query.subscriptions.filter(project=self.project).get()
@fixture
def other_sub(self):
return self.rule.snuba_query.subscriptions.filter(project=self.other_project).get()
@fixture
def rule(self):
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
# Make sure the trigger exists
trigger = create_alert_rule_trigger(rule, CRITICAL_TRIGGER_LABEL, 100)
create_alert_rule_trigger_action(
trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
return rule
@fixture
def trigger(self):
return self.rule.alertruletrigger_set.get()
@fixture
def action(self):
return self.trigger.alertruletriggeraction_set.get()
def build_subscription_update(self, subscription, time_delta=None, value=EMPTY):
if time_delta is not None:
timestamp = timezone.now() + time_delta
else:
timestamp = timezone.now()
timestamp = timestamp.replace(tzinfo=pytz.utc, microsecond=0)
data = {}
if subscription:
data = {"some_col_name": randint(0, 100) if value is EMPTY else value}
values = {"data": [data]}
return {
"subscription_id": subscription.subscription_id if subscription else uuid4().hex,
"values": values,
"timestamp": timestamp,
"interval": 1,
"partition": 1,
"offset": 1,
}
def send_update(self, rule, value, time_delta=None, subscription=None):
self.email_action_handler.reset_mock()
if time_delta is None:
time_delta = timedelta()
if subscription is None:
subscription = self.sub
processor = SubscriptionProcessor(subscription)
message = self.build_subscription_update(subscription, value=value, time_delta=time_delta)
with self.feature(
["organizations:incidents", "organizations:performance-view"]
), self.capture_on_commit_callbacks(execute=True):
processor.process_update(message)
return processor
def assert_slack_calls(self, trigger_labels):
expected = [f"{label}: some rule 2" for label in trigger_labels]
actual = [
json.loads(call_kwargs["data"]["attachments"])[0]["title"]
for (_, call_kwargs) in self.slack_client.call_args_list
]
assert expected == actual
self.slack_client.reset_mock()
def assert_trigger_exists_with_status(self, incident, trigger, status):
assert IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger, status=status.value
).exists()
def assert_trigger_does_not_exist_for_incident(self, incident, trigger):
assert not IncidentTrigger.objects.filter(
incident=incident, alert_rule_trigger=trigger
).exists()
def assert_trigger_does_not_exist(self, trigger, incidents_to_exclude=None):
if incidents_to_exclude is None:
incidents_to_exclude = []
assert (
not IncidentTrigger.objects.filter(alert_rule_trigger=trigger)
.exclude(incident__in=incidents_to_exclude)
.exists()
)
def assert_action_handler_called_with_actions(self, incident, actions, project=None):
project = self.project if project is None else project
if not actions:
if not incident:
assert not self.email_action_handler.called
else:
for call_args in self.email_action_handler.call_args_list:
assert call_args[0][1] != incident
else:
self.email_action_handler.assert_has_calls(
[call(action, incident, project) for action in actions], any_order=True
)
def assert_actions_fired_for_incident(self, incident, actions=None, project=None):
actions = [] if actions is None else actions
project = self.project if project is None else project
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.fire.call_args_list)
def assert_actions_resolved_for_incident(self, incident, actions=None, project=None):
project = self.project if project is None else project
actions = [] if actions is None else actions
self.assert_action_handler_called_with_actions(incident, actions, project)
assert len(actions) == len(self.email_action_handler.return_value.resolve.call_args_list)
def assert_no_active_incident(self, rule, subscription=None):
assert not self.active_incident_exists(rule, subscription=subscription)
def assert_active_incident(self, rule, subscription=None):
incidents = self.active_incident_exists(rule, subscription=subscription)
assert incidents
return incidents[0]
def active_incident_exists(self, rule, subscription=None):
if subscription is None:
subscription = self.sub
return list(
Incident.objects.filter(
type=IncidentType.ALERT_TRIGGERED.value,
alert_rule=rule,
projects=subscription.project,
).exclude(status=IncidentStatus.CLOSED.value)
)
def assert_trigger_counts(self, processor, trigger, alert_triggers=0, resolve_triggers=0):
assert processor.trigger_alert_counts[trigger.id] == alert_triggers
alert_stats, resolve_stats = get_alert_rule_stats(
processor.alert_rule, processor.subscription, [trigger]
)[1:]
assert alert_stats[trigger.id] == alert_triggers
assert resolve_stats[trigger.id] == resolve_triggers
def test_removed_alert_rule(self):
message = self.build_subscription_update(self.sub)
self.rule.delete()
with self.feature(["organizations:incidents", "organizations:performance-view"]):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.no_alert_rule_for_subscription"
)
# TODO: Check subscription is deleted once we start doing that
def test_removed_project(self):
message = self.build_subscription_update(self.sub)
self.project.delete()
with self.feature(["organizations:incidents", "organizations:performance-view"]):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with("incidents.alert_rules.ignore_deleted_project")
def test_no_feature(self):
message = self.build_subscription_update(self.sub)
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.ignore_update_missing_incidents"
)
def test_no_feature_performance(self):
self.sub.snuba_query.dataset = "transactions"
message = self.build_subscription_update(self.sub)
with self.feature("organizations:incidents"):
SubscriptionProcessor(self.sub).process_update(message)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.ignore_update_missing_incidents_performance"
)
def test_skip_already_processed_update(self):
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold)
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=-1))
self.metrics.incr.assert_called_once_with(
"incidents.alert_rules.skipping_already_processed_update"
)
self.metrics.incr.reset_mock()
self.send_update(self.rule, self.trigger.alert_threshold, timedelta(hours=1))
self.metrics.incr.assert_not_called() # NOQA
def test_no_alert(self):
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(self.rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_alert(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
assert incident.date_started == (
timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window)
)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_dedupe(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
c_trigger = self.trigger
c_action_2 = create_alert_rule_trigger_action(
self.trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
w_trigger = create_alert_rule_trigger(
self.rule, WARNING_TRIGGER_LABEL, c_trigger.alert_threshold - 10
)
create_alert_rule_trigger_action(
w_trigger,
AlertRuleTriggerAction.Type.EMAIL,
AlertRuleTriggerAction.TargetType.USER,
str(self.user.id),
)
processor = self.send_update(rule, c_trigger.alert_threshold + 1)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
assert incident.date_started == (
timezone.now().replace(microsecond=0) - timedelta(seconds=rule.snuba_query.time_window)
)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [c_action_2])
def test_alert_nullable(self):
# Verify that an alert rule that only expects a single update to be over the
# alert threshold triggers correctly
rule = self.rule
self.trigger
processor = self.send_update(rule, None)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
def test_alert_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold triggers correctly
rule = self.rule
trigger = self.trigger
rule.update(threshold_period=2)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_alert_multiple_triggers_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be over the
# alert threshold doesn't trigger if there are two updates that are above with
# an update that is below the threshold in the middle
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_no_active_incident_resolve(self):
# Test that we don't track stats for resolving if there are no active incidents
# related to the alert rule.
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, rule.resolve_threshold - 1)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(trigger)
self.assert_action_handler_called_with_actions(None, [])
def test_resolve(self):
# Verify that an alert rule that only expects a single update to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold triggers correctly
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_resolve_multiple_threshold_periods_non_consecutive(self):
# Verify that a rule that expects two consecutive updates to be under the
# resolve threshold doesn't trigger if there's two updates that are below with
# an update that is above the threshold in the middle
rule = self.rule
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-4))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
rule.update(threshold_period=2)
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 1)
self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
def test_auto_resolve(self):
# Verify that we resolve an alert rule automatically even if no resolve
# threshold is set
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_percent_boundary(self):
# Verify that we resolve an alert rule automatically even if no resolve
# threshold is set
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
trigger.update(alert_threshold=0.5)
processor = self.send_update(rule, trigger.alert_threshold + 0.1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_boundary(self):
# Verify that we resolve an alert rule automatically if the value hits the
# original alert trigger value
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_reversed(self):
# Test auto resolving works correctly when threshold is reversed
rule = self.rule
rule.update(resolve_threshold=None, threshold_type=AlertRuleThresholdType.BELOW.value)
trigger = self.trigger
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_auto_resolve_multiple_trigger(self):
# Test auto resolving works correctly when multiple triggers are present.
rule = self.rule
rule.update(resolve_threshold=None)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", trigger.alert_threshold - 10)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(rule, other_trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_reversed_threshold_alert(self):
# Test that inverting thresholds correctly alerts
rule = self.rule
trigger = self.trigger
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
trigger.update(alert_threshold=rule.resolve_threshold + 1)
processor = self.send_update(rule, trigger.alert_threshold + 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_does_not_exist(trigger)
self.assert_action_handler_called_with_actions(None, [])
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
def test_reversed_threshold_resolve(self):
# Test that inverting thresholds correctly resolves
rule = self.rule
trigger = self.trigger
rule.update(threshold_type=AlertRuleThresholdType.BELOW.value)
trigger.update(alert_threshold=rule.resolve_threshold + 1)
processor = self.send_update(rule, trigger.alert_threshold - 1, timedelta(minutes=-3))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
processor = self.send_update(rule, rule.resolve_threshold - 1, timedelta(minutes=-2))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(rule, rule.resolve_threshold + 1, timedelta(minutes=-1))
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
def test_multiple_subscriptions_do_not_conflict(self):
# Verify that multiple subscriptions associated with a rule don't conflict with
# each other
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
# Send an update through for the first subscription. This shouldn't trigger an
# incident, since we need two consecutive updates that are over the threshold.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Have an update come through for the other sub. This shouldn't influence the original
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 1, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger)
self.assert_action_handler_called_with_actions(None, [])
# Send another update through for the first subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_does_not_exist(self.trigger, [incident])
# Send another update through for the second subscription. This should trigger an
# incident for just this subscription.
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(other_incident, [self.action], self.other_project)
# Now we want to test that resolving is isolated. Send another update through
# for the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
other_incident = self.assert_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(other_incident, [])
# This second update for the second subscription should resolve its incident,
# but not the incident from the first subscription.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.other_sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.ACTIVE)
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(other_incident, [self.action], self.other_project)
# This second update for the first subscription should resolve its incident now.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, self.trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
self.assert_no_active_incident(rule, self.other_sub)
self.assert_trigger_exists_with_status(other_incident, self.trigger, TriggerStatus.RESOLVED)
self.assert_action_handler_called_with_actions(other_incident, [])
def test_multiple_triggers(self):
rule = self.rule
rule.update(threshold_period=2)
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 1, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_does_not_exist(trigger)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_action_handler_called_with_actions(None, [])
# This should cause both to increment, although only `trigger` should fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 1, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_does_not_exist(other_trigger)
self.assert_actions_fired_for_incident(incident, [self.action])
# Now only `other_trigger` should increment and fire.
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [other_action])
# Now send through two updates where we're below threshold for the rule. This
# should resolve all triggers and the incident should be closed.
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-7), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 1)
self.assert_trigger_counts(processor, other_trigger, 0, 1)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_action_handler_called_with_actions(incident, [])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-6), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_slack_multiple_triggers_critical_before_warning(self):
"""
Test that ensures that when we get a critical update is sent followed by a warning update,
the warning update is not swallowed and an alert is triggered as a warning alert granted
the count is above the warning trigger threshold
"""
from sentry.incidents.action_handlers import SlackActionHandler
slack_handler = SlackActionHandler
# Create Slack Integration
integration = Integration.objects.create(
provider="slack",
name="Team A",
external_id="TXXXXXXX1",
metadata={
"access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
"installation_type": "born_as_bot",
},
)
integration.add_organization(self.project.organization, self.user)
# Register Slack Handler
AlertRuleTriggerAction.register_type(
"slack",
AlertRuleTriggerAction.Type.SLACK,
[AlertRuleTriggerAction.TargetType.SPECIFIC],
integration_provider="slack",
)(slack_handler)
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule 2",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
trigger = create_alert_rule_trigger(rule, "critical", 100)
trigger_warning = create_alert_rule_trigger(rule, "warning", 10)
for t in [trigger, trigger_warning]:
create_alert_rule_trigger_action(
t,
AlertRuleTriggerAction.Type.SLACK,
AlertRuleTriggerAction.TargetType.SPECIFIC,
integration=integration,
input_channel_id="#workflow",
)
# Send Critical Update
self.send_update(
rule,
trigger.alert_threshold + 5,
timedelta(minutes=-10),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Critical"])
# Send Warning Update
self.send_update(
rule,
trigger_warning.alert_threshold + 5,
timedelta(minutes=0),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Warning"])
self.assert_active_incident(rule)
def test_slack_multiple_triggers_critical_fired_twice_before_warning(self):
"""
Test that ensures that when we get a critical update is sent followed by a warning update,
the warning update is not swallowed and an alert is triggered as a warning alert granted
the count is above the warning trigger threshold
"""
from sentry.incidents.action_handlers import SlackActionHandler
slack_handler = SlackActionHandler
# Create Slack Integration
integration = Integration.objects.create(
provider="slack",
name="Team A",
external_id="TXXXXXXX1",
metadata={
"access_token": "xoxp-xxxxxxxxx-xxxxxxxxxx-xxxxxxxxxxxx",
"installation_type": "born_as_bot",
},
)
integration.add_organization(self.project.organization, self.user)
# Register Slack Handler
AlertRuleTriggerAction.register_type(
"slack",
AlertRuleTriggerAction.Type.SLACK,
[AlertRuleTriggerAction.TargetType.SPECIFIC],
integration_provider="slack",
)(slack_handler)
rule = self.create_alert_rule(
projects=[self.project, self.other_project],
name="some rule 2",
query="",
aggregate="count()",
time_window=1,
threshold_type=AlertRuleThresholdType.ABOVE,
resolve_threshold=10,
threshold_period=1,
)
trigger = create_alert_rule_trigger(rule, "critical", 100)
trigger_warning = create_alert_rule_trigger(rule, "warning", 10)
for t in [trigger, trigger_warning]:
create_alert_rule_trigger_action(
t,
AlertRuleTriggerAction.Type.SLACK,
AlertRuleTriggerAction.TargetType.SPECIFIC,
integration=integration,
input_channel_id="#workflow",
)
self.assert_slack_calls([])
# Send update above critical
self.send_update(
rule,
trigger.alert_threshold + 5,
timedelta(minutes=-10),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls(["Critical"])
# Send second update above critical
self.send_update(
rule,
trigger.alert_threshold + 6,
timedelta(minutes=-9),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_slack_calls([])
# Send update below critical but above warning
self.send_update(
rule,
trigger_warning.alert_threshold + 5,
timedelta(minutes=0),
subscription=rule.snuba_query.subscriptions.filter(project=self.project).get(),
)
self.assert_active_incident(rule)
self.assert_slack_calls(["Warning"])
def test_multiple_triggers_at_same_time(self):
# Check that both triggers fire if an update comes through that exceeds both of
# their thresholds
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action, other_action])
def test_multiple_triggers_resolve_separately(self):
# Check that resolve triggers fire separately
rule = self.rule
trigger = self.trigger
other_trigger = create_alert_rule_trigger(self.rule, "hello", 200)
other_action = create_alert_rule_trigger_action(
other_trigger, AlertRuleTriggerAction.Type.EMAIL, AlertRuleTriggerAction.TargetType.USER
)
processor = self.send_update(
rule, other_trigger.alert_threshold + 1, timedelta(minutes=-10), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.ACTIVE)
self.assert_actions_fired_for_incident(incident, [self.action, other_action])
processor = self.send_update(
rule, other_trigger.alert_threshold - 1, timedelta(minutes=-9), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
incident = self.assert_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.ACTIVE)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [other_action])
processor = self.send_update(
rule, rule.resolve_threshold - 1, timedelta(minutes=-8), subscription=self.sub
)
self.assert_trigger_counts(processor, trigger, 0, 0)
self.assert_trigger_counts(processor, other_trigger, 0, 0)
self.assert_no_active_incident(rule, self.sub)
self.assert_trigger_exists_with_status(incident, trigger, TriggerStatus.RESOLVED)
self.assert_trigger_exists_with_status(incident, other_trigger, TriggerStatus.RESOLVED)
self.assert_actions_resolved_for_incident(incident, [self.action])
