def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
project = data.pop('project')
delete_raw_event(project, event_id)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process', time() - start_time,
instance=data['platform'])
def test_group_release_with_env(self):
manager = EventManager(self.make_event(
release='1.0', environment='prod',
event_id='a' * 32))
event = manager.save(1)
release = Release.objects.get(version='1.0', projects=event.project_id)
assert GroupRelease.objects.filter(
release_id=release.id,
group_id=event.group_id,
environment='prod',
).exists()
manager = EventManager(self.make_event(
release='1.0', environment='staging',
event_id='b' * 32))
event = manager.save(1)
release = Release.objects.get(version='1.0', projects=event.project_id)
assert GroupRelease.objects.filter(
release_id=release.id,
group_id=event.group_id,
environment='staging',
).exists()
def test_long_culprit():
manager = EventManager(make_event(
culprit='x' * (MAX_CULPRIT_LENGTH + 1),
))
manager.normalize()
data = manager.get_data()
assert len(data['culprit']) == MAX_CULPRIT_LENGTH
def test_ephemral_interfaces_removed_on_save(self):
manager = EventManager(self.make_event(platform='python'))
event = manager.save(1)
group = event.group
assert group.platform == 'python'
assert event.platform == 'python'
def test_long_transaction():
manager = EventManager(make_event(
transaction='x' * (MAX_CULPRIT_LENGTH + 1),
))
manager.normalize()
data = manager.get_data()
assert len(data['transaction']) == MAX_CULPRIT_LENGTH
def test_moves_stacktrace_to_exception():
manager = EventManager(
make_event(
exception={
'type': 'MyException',
},
stacktrace={
'frames': [
{
'lineno': 1,
'filename': 'foo.py',
}, {
'lineno': 1,
'filename': 'bar.py',
}
]
}
)
)
manager.normalize()
data = manager.get_data()
frames = data['exception']['values'][0]['stacktrace']['frames']
assert frames[0]['lineno'] == 1
assert frames[0]['filename'] == 'foo.py'
assert frames[1]['lineno'] == 1
assert frames[1]['filename'] == 'bar.py'
assert 'stacktrace' not in data
def test_does_not_unresolve_group(self, plugin_is_regression):
# N.B. EventManager won't unresolve the group unless the event2 has a
# later timestamp than event1. MySQL doesn't support microseconds.
plugin_is_regression.return_value = False
manager = EventManager(self.make_event(
event_id='a' * 32, checksum='a' * 32,
timestamp=1403007314,
))
with self.tasks():
event = manager.save(1)
group = Group.objects.get(id=event.group_id)
group.status = GroupStatus.RESOLVED
group.save()
assert group.is_resolved()
manager = EventManager(self.make_event(
event_id='b' * 32, checksum='a' * 32,
timestamp=1403007315,
))
event2 = manager.save(1)
assert event.group_id == event2.group_id
group = Group.objects.get(id=group.id)
assert group.is_resolved()
def test_long_message(self):
manager = EventManager(self.make_event(
message='x' * (settings.SENTRY_MAX_MESSAGE_LENGTH + 1),
))
data = manager.normalize()
assert len(data['sentry.interfaces.Message']['message']) == \
settings.SENTRY_MAX_MESSAGE_LENGTH
def test_saves_event_mapping_when_sampled(self, should_sample):
should_sample.return_value = True
event_id = 'a' * 32
manager = EventManager(self.make_event(event_id=event_id))
event = manager.save(1)
# This is a brand new event, so it is actually saved.
# In this case, we don't need an EventMapping, but we
# do need the Event.
assert not EventMapping.objects.filter(
group_id=event.group_id,
event_id=event_id,
).exists()
assert Event.objects.filter(
event_id=event_id,
).exists()
event_id = 'b' * 32
manager = EventManager(self.make_event(event_id=event_id))
event = manager.save(1)
# This second is a dupe, so should be sampled
# For a sample, we want to store the EventMapping,
# but don't need to store the Event
assert EventMapping.objects.filter(
group_id=event.group_id,
event_id=event_id,
).exists()
assert not Event.objects.filter(
event_id=event_id,
).exists()
def process_event_from_kafka(message):
project = Project.objects.get_from_cache(pk=message['project_id'])
remote_addr = message['remote_addr']
helper = ClientApiHelper(
agent=message['agent'],
project_id=project.id,
ip_address=remote_addr,
)
helper.context.bind_project(project)
auth = Auth(message['auth'], message['auth'].pop('is_public'))
helper.context.bind_auth(auth)
key = helper.project_key_from_auth(auth)
data = message['data']
version = data['version']
event_manager = EventManager(
data,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
user_agent=helper.context.agent,
version=version,
)
event_manager._normalized = True
del data
return process_event(event_manager, project, key,
remote_addr, helper, attachments=None)
def send(self, **kwargs):
# TODO(dcramer): this should respect rate limits/etc and use the normal
# pipeline
from sentry.app import tsdb
from sentry.coreapi import insert_data_to_database
from sentry.event_manager import EventManager
from sentry.models import Project
try:
project = Project.objects.get_from_cache(id=settings.SENTRY_PROJECT)
except Project.DoesNotExist:
self.error_logger.error('Internal project (id=%s) does not exist',
settings.SENTRY_PROJECT)
return
metrics.incr('events.total', 1)
kwargs['project'] = project.id
try:
manager = EventManager(kwargs)
data = manager.normalize()
tsdb.incr_multi([
(tsdb.models.project_total_received, project.id),
(tsdb.models.organization_total_received, project.organization_id),
])
insert_data_to_database(data)
except Exception as e:
if self.raise_send_errors:
raise
self.error_logger.error(
'Unable to record event: %s\nEvent was: %r', e,
kwargs['message'], exc_info=True)
def test_platform_is_saved(self):
manager = EventManager(self.make_event(platform='python'))
event = manager.save(1)
group = event.group
assert group.platform == 'python'
assert event.platform == 'python'
def test_unresolves_group(self):
ts = time() - 300
# N.B. EventManager won't unresolve the group unless the event2 has a
# later timestamp than event1.
manager = EventManager(
make_event(
event_id='a' * 32,
checksum='a' * 32,
timestamp=ts,
)
)
with self.tasks():
event = manager.save(1)
group = Group.objects.get(id=event.group_id)
group.status = GroupStatus.RESOLVED
group.save()
assert group.is_resolved()
manager = EventManager(
make_event(
event_id='b' * 32,
checksum='a' * 32,
timestamp=ts + 50,
)
)
event2 = manager.save(1)
assert event.group_id == event2.group_id
group = Group.objects.get(id=group.id)
assert not group.is_resolved()
def test_updates_group_with_fingerprint(self):
ts = time() - 200
manager = EventManager(
make_event(
message='foo',
event_id='a' * 32,
fingerprint=['a' * 32],
timestamp=ts,
)
)
with self.tasks():
event = manager.save(1)
manager = EventManager(
make_event(
message='foo bar',
event_id='b' * 32,
fingerprint=['a' * 32],
timestamp=ts,
)
)
with self.tasks():
event2 = manager.save(1)
group = Group.objects.get(id=event.group_id)
assert group.times_seen == 2
assert group.last_seen == event.datetime
assert group.message == event2.message
def test_invalid_environment(self):
manager = EventManager(make_event(**{
'environment': 'bad/name',
}))
manager.normalize()
event = manager.save(self.project.id)
assert dict(event.tags).get('environment') is None
def test_unresolves_group_with_auto_resolve(self, mock_is_resolved):
ts = time() - 100
mock_is_resolved.return_value = False
manager = EventManager(
make_event(
event_id='a' * 32,
checksum='a' * 32,
timestamp=ts,
)
)
with self.tasks():
event = manager.save(1)
mock_is_resolved.return_value = True
manager = EventManager(
make_event(
event_id='b' * 32,
checksum='a' * 32,
timestamp=ts + 100,
)
)
with self.tasks():
event2 = manager.save(1)
assert event.group_id == event2.group_id
group = Group.objects.get(id=event.group.id)
assert group.active_at.replace(second=0) == event2.datetime.replace(second=0)
assert group.active_at.replace(second=0) != event.datetime.replace(second=0)
def test_updates_group(self):
manager = EventManager(
self.make_event(
message='foo',
event_id='a' * 32,
checksum='a' * 32,
)
)
event = manager.save(1)
manager = EventManager(
self.make_event(
message='foo bar',
event_id='b' * 32,
checksum='a' * 32,
)
)
with self.tasks():
event2 = manager.save(1)
group = Group.objects.get(id=event.group_id)
assert group.times_seen == 2
assert group.last_seen.replace(microsecond=0) == event.datetime.replace(microsecond=0)
assert group.message == event2.message
assert group.data.get('type') == 'default'
assert group.data.get('metadata') == {
'title': 'foo bar',
}
def test_updates_group_with_fingerprint(self):
manager = EventManager(
self.make_event(
message='foo',
event_id='a' * 32,
fingerprint=['a' * 32],
)
)
with self.tasks():
event = manager.save(1)
manager = EventManager(
self.make_event(
message='foo bar',
event_id='b' * 32,
fingerprint=['a' * 32],
)
)
with self.tasks():
event2 = manager.save(1)
group = Group.objects.get(id=event.group_id)
assert group.times_seen == 2
assert group.last_seen.replace(microsecond=0) == event.datetime.replace(microsecond=0)
assert group.message == event2.message
def test_unresolves_group_with_auto_resolve(self, mock_is_resolved):
mock_is_resolved.return_value = False
manager = EventManager(
self.make_event(
event_id='a' * 32,
checksum='a' * 32,
timestamp=1403007314,
)
)
with self.tasks():
event = manager.save(1)
mock_is_resolved.return_value = True
manager = EventManager(
self.make_event(
event_id='b' * 32,
checksum='a' * 32,
timestamp=1403007414,
)
)
with self.tasks():
event2 = manager.save(1)
assert event.group_id == event2.group_id
group = Group.objects.get(id=event.group.id)
assert group.active_at == event2.datetime != event.datetime
def test_user_report_gets_environment(self):
project = self.create_project()
environment = Environment.objects.create(
project_id=project.id,
organization_id=project.organization_id,
name='production',
)
environment.add_project(project)
event_id = 'a' * 32
group = self.create_group(project=project)
UserReport.objects.create(
group=group,
project=project,
event_id=event_id,
name='foo',
email='*****@*****.**',
comments='It Broke!!!',
)
manager = EventManager(
self.make_event(
environment=environment.name,
event_id=event_id,
group=group))
manager.normalize()
manager.save(project.id)
assert UserReport.objects.get(event_id=event_id).environment == environment
def test(self, mock_delay_index_event_tags, mock_eventstream_insert):
now = datetime.utcnow()
def _get_event_count():
return snuba.query(
start=now - timedelta(days=1),
end=now + timedelta(days=1),
groupby=['project_id'],
filter_keys={'project_id': [self.project.id]},
).get(self.project.id, 0)
assert _get_event_count() == 0
raw_event = {
'event_id': 'a' * 32,
'message': 'foo',
'timestamp': time.mktime(now.timetuple()),
'level': logging.ERROR,
'logger': 'default',
'tags': [],
}
manager = EventManager(raw_event)
manager.normalize()
event = manager.save(self.project.id)
# verify eventstream was called by EventManager
insert_args, insert_kwargs = list(mock_eventstream_insert.call_args)
assert not insert_args
assert insert_kwargs == {
'event': event,
'group': event.group,
'is_new_group_environment': True,
'is_new': True,
'is_regression': False,
'is_sample': False,
'primary_hash': 'acbd18db4cc2f85cedef654fccc4a4d8',
'skip_consume': False
}
assert mock_delay_index_event_tags.call_count == 1
# pass arguments on to Kafka EventManager
self.kafka_eventstream.insert(*insert_args, **insert_kwargs)
produce_args, produce_kwargs = list(self.kafka_eventstream.producer.produce.call_args)
assert not produce_args
assert produce_kwargs['topic'] == 'events'
assert produce_kwargs['key'] == six.text_type(self.project.id)
version, type_, payload1, payload2 = json.loads(produce_kwargs['value'])
assert version == 2
assert type_ == 'insert'
# insert what would have been the Kafka payload directly
# into Snuba, expect an HTTP 200 and for the event to now exist
snuba_eventstream = SnubaEventStream()
snuba_eventstream._send(self.project.id, 'insert', (payload1, payload2))
assert _get_event_count() == 1
def test_does_default_ip_address_to_user(self):
manager = EventManager(
self.make_event(
**{"sentry.interfaces.Http": {"url": "http://example.com", "env": {"REMOTE_ADDR": "127.0.0.1"}}}
)
)
data = manager.normalize()
assert data["sentry.interfaces.User"]["ip_address"] == "127.0.0.1"
def test_culprit_is_not_transaction(self):
manager = EventManager(make_event(
culprit='foobar',
))
manager.normalize()
event1 = manager.save(1)
assert event1.transaction is None
assert event1.culprit == 'foobar'
def test_removes_some_empty_containers(key, value):
event = make_event()
event[key] = value
manager = EventManager(event)
manager.normalize()
data = manager.get_data()
assert key not in data
def test_broken_regression_signal(self, send):
send.side_effect = Exception()
manager = EventManager(self.make_event())
event = manager.save(1)
assert event.message == 'foo'
assert event.project_id == 1
def test_environment(self):
manager = EventManager(self.make_event(**{
'environment': 'beta',
}))
manager.normalize()
event = manager.save(self.project.id)
assert dict(event.tags).get('environment') == 'beta'
def save_event():
manager = EventManager(self.make_event(**{
'event_id': uuid.uuid1().hex, # don't deduplicate
'environment': 'beta',
'release': release_version,
}))
manager.normalize()
return manager.save(self.project.id)
def test_transaction_as_culprit(self):
manager = EventManager(make_event(
transaction='foobar',
))
manager.normalize()
event = manager.save(1)
assert event.transaction == 'foobar'
assert event.culprit == 'foobar'
def test_invalid_transaction(self):
dict_input = {'messages': 'foo'}
manager = EventManager(self.make_event(
transaction=dict_input,
))
manager.normalize()
event = manager.save(1)
assert event.transaction is None
def inner(data):
mgr = EventManager(data={"sdk": data})
mgr.normalize()
evt = Event(data=mgr.get_data())
insta_snapshot({
'errors': evt.data.get('errors'),
'to_json': evt.interfaces.get('sdk').to_json()
})
def test_marks_as_unresolved_only_with_new_release(self, plugin_is_regression):
plugin_is_regression.return_value = True
old_release = Release.objects.create(
version='a',
project=self.project,
date_added=timezone.now() - timedelta(minutes=30),
)
manager = EventManager(self.make_event(
event_id='a' * 32,
checksum='a' * 32,
timestamp=time() - 50000, # need to work around active_at
release=old_release.version,
))
event = manager.save(1)
group = event.group
group.update(status=GroupStatus.RESOLVED)
resolution = GroupResolution.objects.create(
release=old_release,
group=group,
)
activity = Activity.objects.create(
group=group,
project=group.project,
type=Activity.SET_RESOLVED_IN_RELEASE,
ident=resolution.id,
data={'version': ''},
)
manager = EventManager(self.make_event(
event_id='b' * 32,
checksum='a' * 32,
timestamp=time(),
release=old_release.version,
))
event = manager.save(1)
assert event.group_id == group.id
group = Group.objects.get(id=group.id)
assert group.status == GroupStatus.RESOLVED
activity = Activity.objects.get(id=activity.id)
assert activity.data['version'] == ''
assert GroupResolution.objects.filter(group=group).exists()
manager = EventManager(self.make_event(
event_id='c' * 32,
checksum='a' * 32,
timestamp=time(),
release='b',
))
event = manager.save(1)
assert event.group_id == group.id
group = Group.objects.get(id=group.id)
assert group.status == GroupStatus.UNRESOLVED
activity = Activity.objects.get(id=activity.id)
assert activity.data['version'] == 'b'
assert not GroupResolution.objects.filter(group=group).exists()
assert Activity.objects.filter(
group=group,
type=Activity.SET_REGRESSION,
).exists()
def test_long_message(self):
manager = EventManager(self.make_event(
message='x' * (settings.SENTRY_MAX_MESSAGE_LENGTH + 1),
))
data = manager.normalize()
assert len(data['message']) == settings.SENTRY_MAX_MESSAGE_LENGTH
def test_event_user(self):
manager = EventManager(make_event(
event_id='a',
environment='totally unique environment',
**{'sentry.interfaces.User': {
'id': '1',
}}
))
manager.normalize()
with self.tasks():
event = manager.save(self.project.id)
environment_id = Environment.get_for_organization_id(
event.project.organization_id,
'totally unique environment',
).id
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_group,
(event.group.id, ),
event.datetime,
event.datetime,
) == {
event.group.id: 1,
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_project,
(event.project.id, ),
event.datetime,
event.datetime,
) == {
event.project.id: 1,
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_group,
(event.group.id, ),
event.datetime,
event.datetime,
environment_id=environment_id,
) == {
event.group.id: 1,
}
assert tsdb.get_distinct_counts_totals(
tsdb.models.users_affected_by_project,
(event.project.id, ),
event.datetime,
event.datetime,
environment_id=environment_id,
) == {
event.project.id: 1,
}
euser = EventUser.objects.get(
project_id=self.project.id,
ident='1',
)
assert event.get_tag('sentry:user') == euser.tag_value
# ensure event user is mapped to tags in second attempt
manager = EventManager(
make_event(
event_id='b',
**{'sentry.interfaces.User': {
'id': '1',
'name': 'jane',
}}
)
)
manager.normalize()
with self.tasks():
event = manager.save(self.project.id)
euser = EventUser.objects.get(id=euser.id)
assert event.get_tag('sentry:user') == euser.tag_value
assert euser.name == 'jane'
assert euser.ident == '1'
def validate_and_normalize(self, data, request_env=None):
data = self.helper.validate_data(data)
return EventManager(data).normalize(request_env=request_env)
def test_inferred_culprit_from_empty_stacktrace(self):
manager = EventManager(make_event(stacktrace={"frames": []}))
manager.normalize()
event = manager.save(1)
assert event.culprit == ''
def send(self, **kwargs):
# TODO(dcramer): this should respect rate limits/etc and use the normal
# pipeline
# Report the issue to an upstream Sentry if active
# NOTE: we don't want to check self.is_enabled() like normal, since
# is_enabled behavior is overridden in this class. We explicitly
# want to check if the remote is active.
if self.remote.is_active():
from sentry import options
# Append some extra tags that are useful for remote reporting
super_kwargs = copy.deepcopy(kwargs)
super_kwargs['tags']['install-id'] = options.get('sentry:install-id')
super(SentryInternalClient, self).send(**super_kwargs)
if not is_current_event_safe():
return
from sentry.app import tsdb
from sentry.coreapi import ClientApiHelper
from sentry.event_manager import EventManager
from sentry.models import Project
helper = ClientApiHelper(
agent='raven-python/%s (sentry %s)' % (raven.VERSION, sentry.VERSION),
project_id=settings.SENTRY_PROJECT,
version=self.protocol_version,
)
try:
project = Project.objects.get_from_cache(id=settings.SENTRY_PROJECT)
except DatabaseError:
self.error_logger.error('Unable to fetch internal project',
exc_info=True)
return
except Project.DoesNotExist:
self.error_logger.error('Internal project (id=%s) does not exist',
settings.SENTRY_PROJECT)
return
except Exception:
self.error_logger.error(
'Unable to fetch internal project for some unknown reason',
exc_info=True)
return
helper.context.bind_project(project)
metrics.incr('events.total')
kwargs['project'] = project.id
try:
manager = EventManager(kwargs)
data = manager.normalize()
tsdb.incr_multi([
(tsdb.models.project_total_received, project.id),
(tsdb.models.organization_total_received, project.organization_id),
])
helper.insert_data_to_database(data)
except Exception as e:
if self.raise_send_errors:
raise
self.error_logger.error(
'Unable to record event: %s\nEvent was: %r', e,
kwargs['message'], exc_info=True)
def from_kwargs(self, project, **kwargs):
from sentry.event_manager import EventManager
manager = EventManager(kwargs)
manager.normalize()
return manager.save(project)
def digest(request):
random = get_random(request)
# TODO: Refactor all of these into something more manageable.
org = Organization(id=1, slug="example", name="Example Organization")
project = Project(id=1,
slug="example",
name="Example Project",
organization=org)
rules = {
i: Rule(id=i, project=project, label="Rule #%s" % (i, ))
for i in range(1, random.randint(2, 4))
}
state = {
"project": project,
"groups": {},
"rules": rules,
"event_counts": {},
"user_counts": {},
}
records = []
group_generator = make_group_generator(random, project)
for i in range(random.randint(1, 30)):
group = next(group_generator)
state["groups"][group.id] = group
offset = timedelta(seconds=0)
for i in range(random.randint(1, 10)):
offset += timedelta(seconds=random.random() * 120)
data = dict(load_data("python"))
data["message"] = group.message
data.pop("logentry", None)
event_manager = EventManager(data)
event_manager.normalize()
data = event_manager.get_data()
data["timestamp"] = random.randint(to_timestamp(group.first_seen),
to_timestamp(group.last_seen))
event = eventstore.create_event(event_id=uuid.uuid4().hex,
group_id=group.id,
project_id=project.id,
data=data.data)
records.append(
Record(
event.event_id,
Notification(
event,
random.sample(list(state["rules"].keys()),
random.randint(1, len(state["rules"]))),
),
to_timestamp(event.datetime),
))
state["event_counts"][group.id] = random.randint(10, 1e4)
state["user_counts"][group.id] = random.randint(10, 1e4)
digest = build_digest(project, records, state)
start, end, counts = get_digest_metadata(digest)
context = {
"project": project,
"counts": counts,
"digest": digest,
"start": start,
"end": end,
"referrer": "digest_email",
}
add_unsubscribe_link(context)
return MailPreview(
html_template="sentry/emails/digests/body.html",
text_template="sentry/emails/digests/body.txt",
context=context,
).render(request)
def test_interface_is_relabeled(self):
manager = EventManager(self.make_event(user={'id': '1'}))
data = manager.normalize()
assert data['sentry.interfaces.User'] == {'id': '1'}
assert 'user' not in data
def test_long_culprit(self):
manager = EventManager(
self.make_event(culprit='x' * (MAX_CULPRIT_LENGTH + 1), ))
data = manager.normalize()
assert len(data['culprit']) == MAX_CULPRIT_LENGTH
def test_long_transaction(self):
manager = EventManager(
self.make_event(transaction='x' * (MAX_CULPRIT_LENGTH + 1), ))
data = manager.normalize()
assert len(data['transaction']) == MAX_CULPRIT_LENGTH
def test_marks_as_unresolved_with_new_release_with_integration(
self,
plugin_is_regression,
mock_send_activity_notifications_delay,
mock_sync_status_outbound,
):
plugin_is_regression.return_value = True
old_release = Release.objects.create(
version="a",
organization_id=self.project.organization_id,
date_added=timezone.now() - timedelta(minutes=30),
)
old_release.add_project(self.project)
manager = EventManager(
make_event(
event_id="a" * 32,
checksum="a" * 32,
timestamp=time() - 50000, # need to work around active_at
release=old_release.version,
))
event = manager.save(1)
group = event.group
org = group.organization
integration = Integration.objects.create(provider="example",
name="Example")
integration.add_organization(org, self.user)
OrganizationIntegration.objects.filter(
integration_id=integration.id,
organization_id=group.organization.id).update(
config={
"sync_comments": True,
"sync_status_outbound": True,
"sync_status_inbound": True,
"sync_assignee_outbound": True,
"sync_assignee_inbound": True,
})
external_issue = ExternalIssue.objects.get_or_create(
organization_id=org.id,
integration_id=integration.id,
key="APP-%s" % group.id)[0]
GroupLink.objects.get_or_create(
group_id=group.id,
project_id=group.project_id,
linked_type=GroupLink.LinkedType.issue,
linked_id=external_issue.id,
relationship=GroupLink.Relationship.references,
)[0]
group.update(status=GroupStatus.RESOLVED)
resolution = GroupResolution.objects.create(release=old_release,
group=group)
activity = Activity.objects.create(
group=group,
project=group.project,
type=Activity.SET_RESOLVED_IN_RELEASE,
ident=resolution.id,
data={"version": ""},
)
manager = EventManager(
make_event(event_id="b" * 32,
checksum="a" * 32,
timestamp=time(),
release=old_release.version))
with self.tasks():
with self.feature({"organizations:integrations-issue-sync": True}):
event = manager.save(1)
assert event.group_id == group.id
group = Group.objects.get(id=group.id)
assert group.status == GroupStatus.RESOLVED
activity = Activity.objects.get(id=activity.id)
assert activity.data["version"] == ""
assert GroupResolution.objects.filter(group=group).exists()
manager = EventManager(
make_event(event_id="c" * 32,
checksum="a" * 32,
timestamp=time(),
release="b"))
event = manager.save(1)
mock_sync_status_outbound.assert_called_once_with(
external_issue, False, event.group.project_id)
assert event.group_id == group.id
group = Group.objects.get(id=group.id)
assert group.status == GroupStatus.UNRESOLVED
activity = Activity.objects.get(id=activity.id)
assert activity.data["version"] == "b"
assert not GroupResolution.objects.filter(group=group).exists()
activity = Activity.objects.get(group=group,
type=Activity.SET_REGRESSION)
mock_send_activity_notifications_delay.assert_called_once_with(
activity.id)
def test_default_version(self):
manager = EventManager(self.make_event())
data = manager.normalize()
assert data['version'] == '5'
def _do_save_event(
cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs
):
"""
Saves an event to the database.
"""
set_current_project(project_id)
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer("tasks.store.do_save_event.get_cache") as metric_tags:
data = event_processing_store.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get("type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
set_current_project(project_id)
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "post"}, skip_internal=False
)
return
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
manager.save(
project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key
)
except HashDiscarded:
pass
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
event_processing_store.delete_by_key(cache_key)
with metrics.timer("tasks.store.do_save_event.delete_attachment_cache"):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
"events.time-to-process", time() - start_time, instance=data["platform"]
)
def test_tags_as_dict(self):
manager = EventManager(self.make_event(tags={'foo': 'bar'}))
data = manager.normalize()
assert data['tags'] == [('foo', 'bar')]
def test_default_fingerprint(self):
manager = EventManager(make_event())
manager.normalize()
event = manager.save(self.project.id)
assert event.data.get('fingerprint') == ['{{ default }}']
def alert(request):
platform = request.GET.get("platform", "python")
org = Organization(id=1, slug="example", name="Example")
project = Project(id=1, slug="example", name="Example", organization=org)
random = get_random(request)
group = next(make_group_generator(random, project))
data = dict(load_data(platform))
data["message"] = group.message
data["event_id"] = "44f1419e73884cd2b45c79918f4b6dc4"
data.pop("logentry", None)
data["environment"] = "prod"
data["tags"] = [
("logger", "javascript"),
("environment", "prod"),
("level", "error"),
("device", "Other"),
]
event_manager = EventManager(data)
event_manager.normalize()
data = event_manager.get_data()
event = event_manager.save(project.id)
# Prevent CI screenshot from constantly changing
event.data["timestamp"] = 1504656000.0 # datetime(2017, 9, 6, 0, 0)
event_type = get_event_type(event.data)
group.message = event.search_message
group.data = {
"type": event_type.key,
"metadata": event_type.get_metadata(data)
}
rule = Rule(label="An example rule")
# XXX: this interface_list code needs to be the same as in
# src/sentry/mail/adapter.py
interface_list = []
for interface in six.itervalues(event.interfaces):
body = interface.to_email_html(event)
if not body:
continue
text_body = interface.to_string(event)
interface_list.append(
(interface.get_title(), mark_safe(body), text_body))
return MailPreview(
html_template="sentry/emails/error.html",
text_template="sentry/emails/error.txt",
context={
"rule":
rule,
"group":
group,
"event":
event,
"link":
"http://example.com/link",
"interfaces":
interface_list,
"tags":
event.tags,
"project_label":
project.slug,
"commits": [{
# TODO(dcramer): change to use serializer
"repository": {
"status": "active",
"name": "Example Repo",
"url": "https://github.com/example/example",
"dateCreated": "2018-02-28T23:39:22.402Z",
"provider": {
"id": "github",
"name": "GitHub"
},
"id": "1",
},
"score": 2,
"subject": "feat: Do something to raven/base.py",
"message":
"feat: Do something to raven/base.py\naptent vivamus vehicula tempus volutpat hac tortor",
"id": "1b17483ffc4a10609e7921ee21a8567bfe0ed006",
"shortId": "1b17483",
"author": {
"username":
"******",
"isManaged":
False,
"lastActive":
"2018-03-01T18:25:28.149Z",
"id":
"1",
"isActive":
True,
"has2fa":
False,
"name":
"*****@*****.**",
"avatarUrl":
"https://secure.gravatar.com/avatar/51567a4f786cd8a2c41c513b592de9f9?s=32&d=mm",
"dateJoined":
"2018-02-27T22:04:32.847Z",
"emails": [{
"is_verified": False,
"id": "1",
"email": "*****@*****.**"
}],
"avatar": {
"avatarUuid": None,
"avatarType": "letter_avatar"
},
"lastLogin":
"******",
"email":
"*****@*****.**",
},
}],
},
).render(request)
def process(self, request, project, auth, data, **kwargs):
event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited, project=project)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
raise APIRateLimited(rate_limit.retry_after)
result = plugins.first('has_perm', request.user, 'create_event', project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e)))
# mutates data
manager = EventManager(data, version=auth.version)
data = manager.normalize()
# insert IP address if not available
if auth.is_public:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# TODO(dcramer): ideally we'd only validate this if the event_id was
# supplied by the user
cache_key = 'ev:%s:%s' % (project.id, event_id,)
if cache.get(cache_key) is not None:
logger.warning('Discarded recent duplicate event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id)
raise InvalidRequest('An event with the same ID already exists.')
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter()
inst.apply(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
cache.set(cache_key, '', 60 * 5)
logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id)
return event_id
def process(self,
request,
project,
key,
auth,
helper,
data,
attachments=None,
**kwargs):
metrics.incr('events.total')
if not data:
raise APIError('No JSON data was found')
remote_addr = request.META['REMOTE_ADDR']
event_manager = EventManager(
data,
project=project,
key=key,
auth=auth,
client_ip=remote_addr,
user_agent=helper.context.agent,
version=auth.version,
content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''),
)
del data
self.pre_normalize(event_manager, helper)
event_manager.normalize()
agent = request.META.get('HTTP_USER_AGENT')
# TODO: Some form of coordination between the Kafka consumer
# and this method (the 'relay') to decide whether a 429 should
# be returned here.
# Everything before this will eventually be done in the relay.
if (kafka_publisher is not None and not attachments
and random.random() < options.get('store.kafka-sample-rate')):
process_in_kafka = options.get('store.process-in-kafka')
try:
kafka_publisher.publish(
channel=getattr(settings, 'KAFKA_EVENTS_PUBLISHER_TOPIC',
'store-events'),
# Relay will (eventually) need to produce a Kafka message
# with this JSON format.
value=json.dumps({
'data': event_manager.get_data(),
'project_id': project.id,
'auth': {
'sentry_client': auth.client,
'sentry_version': auth.version,
'sentry_secret': auth.secret_key,
'sentry_key': auth.public_key,
'is_public': auth.is_public,
},
'remote_addr': remote_addr,
'agent': agent,
# Whether or not the Kafka consumer is in charge
# of actually processing this event.
'should_process': process_in_kafka,
}))
except Exception as e:
logger.exception("Cannot publish event to Kafka: {}".format(
e.message))
else:
if process_in_kafka:
# This event will be processed by the Kafka consumer, so we
# shouldn't double process it here.
return event_manager.get_data()['event_id']
# Everything after this will eventually be done in a Kafka consumer.
return process_event(event_manager, project, key, remote_addr, helper,
attachments)
def get_normalized_event(data, project):
mgr = EventManager(data, project=project)
mgr.normalize()
return dict(mgr.get_data())
def validate_and_normalize(report, client_ip=None):
manager = EventManager(report, client_ip=client_ip)
manager.normalize()
return manager.get_data()
def make_release_event(self, release_name, project_id):
manager = EventManager(make_event(release=release_name))
manager.normalize()
event = manager.save(project_id)
return event
def test_dupe_message_id(self):
event_id = 'a' * 32
manager = EventManager(make_event(event_id=event_id))
manager.normalize()
manager.save(1)
assert Event.objects.count() == 1
# ensure that calling it again doesn't raise a db error
manager = EventManager(make_event(event_id=event_id))
manager.normalize()
manager.save(1)
assert Event.objects.count() == 1
def validate_and_normalize(data):
manager = EventManager(data)
manager.normalize()
return manager.get_data()
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
from sentry.ingest.outcomes_consumer import mark_signal_sent
event_type = "none"
if cache_key and data is None:
with metrics.timer(
"tasks.store.do_save_event.get_cache") as metric_tags:
data = default_cache.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get(
"type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
key_id = None if data is None else data.get("key_id")
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
event = manager.save(project_id,
assume_normalized=True,
cache_key=cache_key)
with metrics.timer("tasks.store.do_save_event.track_outcome"):
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id,
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
# There is no signal supposed to be sent for this particular
# outcome-reason combination. Prevent the outcome consumer from
# emitting it for now.
#
# XXX(markus): Revisit decision about signals once outcomes consumer is stable.
mark_signal_sent(project_id, event_id)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id,
)
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
default_cache.delete(cache_key)
with metrics.timer(
"tasks.store.do_save_event.delete_attachment_cache"):
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or features.has(
"organizations:event-attachments",
event.project.organization,
actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])
def test_key_id_remains_in_data(self):
manager = EventManager(make_event(key_id=12345))
manager.normalize()
assert manager.get_data()['key_id'] == 12345
event = manager.save(1)
assert event.data['key_id'] == 12345
def alert(request):
platform = request.GET.get('platform', 'python')
org = Organization(
id=1,
slug='example',
name='Example',
)
project = Project(
id=1,
slug='example',
name='Example',
organization=org,
)
random = get_random(request)
group = next(make_group_generator(random, project), )
data = dict(load_data(platform))
data['message'] = group.message
data['event_id'] = '44f1419e73884cd2b45c79918f4b6dc4'
data.pop('logentry', None)
data['environment'] = 'prod'
data['tags'] = [('logger', 'javascript'), ('environment', 'prod'),
('level', 'error'), ('device', 'Other')]
event_manager = EventManager(data)
event_manager.normalize()
data = event_manager.get_data()
event = event_manager.save(project.id)
# Prevent Percy screenshot from constantly changing
event.datetime = datetime(2017, 9, 6, 0, 0)
event.save()
event_type = event_manager.get_event_type()
group.message = event_manager.get_search_message()
group.data = {
'type': event_type.key,
'metadata': event_type.get_metadata(data),
}
rule = Rule(label="An example rule")
interface_list = []
for interface in six.itervalues(event.interfaces):
body = interface.to_email_html(event)
if not body:
continue
interface_list.append((interface.get_title(), mark_safe(body)))
return MailPreview(
html_template='sentry/emails/error.html',
text_template='sentry/emails/error.txt',
context={
'rule':
rule,
'group':
group,
'event':
event,
'link':
'http://example.com/link',
'interfaces':
interface_list,
'tags':
event.get_tags(),
'project_label':
project.slug,
'commits': [{
# TODO(dcramer): change to use serializer
"repository": {
"status": "active",
"name": "Example Repo",
"url": "https://github.com/example/example",
"dateCreated": "2018-02-28T23:39:22.402Z",
"provider": {
"id": "github",
"name": "GitHub"
},
"id": "1"
},
"score": 2,
"subject": "feat: Do something to raven/base.py",
"message":
"feat: Do something to raven/base.py\naptent vivamus vehicula tempus volutpat hac tortor",
"id": "1b17483ffc4a10609e7921ee21a8567bfe0ed006",
"shortId": "1b17483",
"author": {
"username":
"******",
"isManaged":
False,
"lastActive":
"2018-03-01T18:25:28.149Z",
"id":
"1",
"isActive":
True,
"has2fa":
False,
"name":
"*****@*****.**",
"avatarUrl":
"https://secure.gravatar.com/avatar/51567a4f786cd8a2c41c513b592de9f9?s=32&d=mm",
"dateJoined":
"2018-02-27T22:04:32.847Z",
"emails": [{
"is_verified": False,
"id": "1",
"email": "*****@*****.**"
}],
"avatar": {
"avatarUuid": None,
"avatarType": "letter_avatar"
},
"lastLogin":
"******",
"email":
"*****@*****.**"
}
}],
},
).render(request)
def test_bad_logger(self):
manager = EventManager(self.make_event(logger='foo bar'))
data = manager.normalize()
assert data['logger'] == DEFAULT_LOGGER_NAME
def test_marks_as_unresolved_with_new_release(
self, plugin_is_regression,
mock_send_activity_notifications_delay):
plugin_is_regression.return_value = True
old_release = Release.objects.create(
version="a",
organization_id=self.project.organization_id,
date_added=timezone.now() - timedelta(minutes=30),
)
old_release.add_project(self.project)
manager = EventManager(
make_event(
event_id="a" * 32,
checksum="a" * 32,
timestamp=time() - 50000, # need to work around active_at
release=old_release.version,
))
event = manager.save(1)
group = event.group
group.update(status=GroupStatus.RESOLVED)
resolution = GroupResolution.objects.create(release=old_release,
group=group)
activity = Activity.objects.create(
group=group,
project=group.project,
type=Activity.SET_RESOLVED_IN_RELEASE,
ident=resolution.id,
data={"version": ""},
)
manager = EventManager(
make_event(event_id="b" * 32,
checksum="a" * 32,
timestamp=time(),
release=old_release.version))
event = manager.save(1)
assert event.group_id == group.id
group = Group.objects.get(id=group.id)
assert group.status == GroupStatus.RESOLVED
activity = Activity.objects.get(id=activity.id)
assert activity.data["version"] == ""
assert GroupResolution.objects.filter(group=group).exists()
manager = EventManager(
make_event(event_id="c" * 32,
checksum="a" * 32,
timestamp=time(),
release="b"))
event = manager.save(1)
assert event.group_id == group.id
group = Group.objects.get(id=group.id)
assert group.status == GroupStatus.UNRESOLVED
activity = Activity.objects.get(id=activity.id)
assert activity.data["version"] == "b"
assert not GroupResolution.objects.filter(group=group).exists()
activity = Activity.objects.get(group=group,
type=Activity.SET_REGRESSION)
mock_send_activity_notifications_delay.assert_called_once_with(
activity.id)
def test_explicit_version(self):
manager = EventManager(self.make_event(), '6')
data = manager.normalize()
assert data['version'] == '6'
