def test_context_with_only_app_frames(make_exception_snapshot):
values = [{
'type': 'ValueError',
'value': 'hello world',
'module': 'foo.bar',
'stacktrace': {
'frames': [{
'filename': 'foo/baz.py',
'lineno': 1,
'in_app': True,
}]
},
}, {
'type': 'ValueError',
'value': 'hello world',
'module': 'foo.bar',
'stacktrace': {
'frames': [{
'filename': 'foo/baz.py',
'lineno': 1,
'in_app': True,
}]
},
}]
exc = dict(values=values)
normalize_stacktraces_for_grouping({'exception': exc})
make_exception_snapshot(exc)
def test_normalize_with_system_frames(self):
data = {
'stacktrace': {
'frames': [
None,
{
'abs_path': 'http://example.com/foo.js',
'filename': 'foo.js',
'lineno': 4,
'colno': 0,
'in_app': True,
},
{
'abs_path': 'http://example.com/foo.js',
'filename': 'foo.js',
'lineno': 1,
'colno': 0,
},
]
}
}
normalize_stacktraces_for_grouping(data)
assert data['stacktrace']['frames'][1]['in_app'] is True
assert data['stacktrace']['frames'][2]['in_app'] is False
def tes_macos_package_in_app_detection(self):
data = {
"platform": "cocoa",
"debug_meta": {
"images": [] # omitted
},
"exception": {
"values": [{
"stacktrace": {
"frames": [
{
"function": "-[CRLCrashAsyncSafeThread crash]",
"package":
"/Users/haza/Library/Developer/Xcode/Archives/2017-06-19/CrashProbe 19-06-2017, 08.53.xcarchive/Products/Applications/CrashProbe.app/Contents/Frameworks/CrashLib.framework/Versions/A/CrashLib",
"instruction_addr": 4295098388
},
{
"function": "[KSCrash ]",
"package": "/usr/lib/system/libdyld.dylib",
"instruction_addr": 4295098388,
},
]
},
"type": "NSRangeException",
}]
},
"contexts": {
"os": {
"version": "10.12.5",
"type": "os",
"name": "macOS"
}
},
}
config = load_grouping_config(get_default_grouping_config_dict())
normalize_stacktraces_for_grouping(data, grouping_config=config)
frames = data['exception']['values'][0]['stacktrace']['frames']
assert frames[0]['in_app'] is True
assert frames[1]['in_app'] is False
def test_event_hash_variant(insta_snapshot, config_name, test_name, log):
with open(os.path.join(_fixture_path, test_name + '.json')) as f:
input = json.load(f)
# Cutomize grouping config from the _grouping config
grouping_config = get_default_grouping_config_dict(config_name)
grouping_info = input.pop('_grouping', None) or {}
enhancement_base = grouping_info.get('enhancement_base')
enhancements = grouping_info.get('enhancements')
if enhancement_base or enhancements:
enhancement_bases = [enhancement_base] if enhancement_base else []
e = Enhancements.from_config_string(
enhancements or '', bases=enhancement_bases)
grouping_config['enhancements'] = e.dumps()
# Noramlize the event
mgr = EventManager(data=input, grouping_config=grouping_config)
mgr.normalize()
data = mgr.get_data()
# Normalize the stacktrace for grouping. This normally happens in
# save()
normalize_stacktraces_for_grouping(data, load_grouping_config(grouping_config))
evt = Event(data=data, platform=data['platform'])
# Make sure we don't need to touch the DB here because this would
# break stuff later on.
evt.project = None
rv = []
for (key, value) in sorted(evt.get_grouping_variants().items()):
if rv:
rv.append('-' * 74)
rv.append('%s:' % key)
dump_variant(value, rv, 1)
output = '\n'.join(rv)
log(repr(evt.get_hashes()))
assert evt.get_grouping_config() == grouping_config
insta_snapshot(output)
def test_ios_package_in_app_detection(self):
data = {
'platform': 'native',
'stacktrace': {
'frames': [
{
'package':
'/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest',
'instruction_addr': '0x1000'
},
{
'package':
'/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/Frameworks/foo.dylib',
'instruction_addr': '0x2000'
},
{
'package':
'/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/Frameworks/libswiftCore.dylib',
'instruction_addr': '0x3000'
},
{
'package': '/usr/lib/whatever.dylib',
'instruction_addr': '0x4000'
},
]
}
}
config = load_grouping_config(get_default_grouping_config_dict())
normalize_stacktraces_for_grouping(data, grouping_config=config)
# App object should be in_app
assert data['stacktrace']['frames'][0]['in_app'] is True
# Framework should be in app (but optional)
assert data['stacktrace']['frames'][1]['in_app'] is True
# libswift should not be system
assert data['stacktrace']['frames'][2]['in_app'] is False
# Unknown object should default to not in_app
assert data['stacktrace']['frames'][3]['in_app'] is False
def save(self, project_id, raw=False, assume_normalized=False):
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
# Make sure we cache on the project before returning
event._project_cache = project
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
})
return event
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get('level')
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get('transaction')
logger_name = data.get('logger')
release = data.get('release')
dist = data.get('dist')
environment = data.get('environment')
recorded_timestamp = data.get('timestamp')
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, 'tags', value=[])
set_tag(data, 'level', level)
if logger_name:
set_tag(data, 'logger', logger_name)
if environment:
set_tag(data, 'environment', environment)
if transaction_name:
set_tag(data, 'transaction', transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, 'release')
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
set_tag(data, 'sentry:release', release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, 'dist')
set_tag(data, 'sentry:dist', dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, 'user')
set_tag(data, 'sentry:user', event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project))
normalize_stacktraces_for_grouping(data, grouping_config)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right paramters.
data['fingerprint'] = data.get('fingerprint') or ['{{ default }}']
apply_server_fingerprinting(
data, get_fingerprinting_config_for_project(project))
hashes = event.get_hashes()
data['hashes'] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overriden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
event_metadata = materialized_metadata['metadata']
data.update(materialized_metadata)
data['culprit'] = culprit
# index components into ``Event.message``
# See GH-3248
event.message = self.get_search_message(event_metadata, culprit)
received_timestamp = event.data.get('received') or float(
event.datetime.strftime('%s'))
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata['last_received'] = received_timestamp
kwargs = {
'platform': platform,
'message': event.message,
'culprit': culprit,
'logger': logger_name,
'level': LOG_LEVELS_MAP.get(level),
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': group_metadata,
}
if release:
kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(
using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project,
group=group,
event_id=event_id)
except IntegrityError:
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
})
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release': release if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
}))
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
logger.info('duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
})
return event
tagstore.delay_index_event_tags(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=event.tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
})
if is_new_group_environment:
buffer.incr(ReleaseProjectEnvironment, {'new_issues_count': 1},
{
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
})
safe_execute(Group.objects.add_tags,
group,
environment,
event.get_tags(),
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project,
group=group,
sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
metrics.timing('events.size.data.post_save',
event.size,
tags={'project_id': project.id})
return event
def test_ios_function_name_in_app_detection(self):
data = {
"platform": "cocoa",
"debug_meta": {
"images": [] # omitted
},
"exception": {
"values": [{
"stacktrace": {
"frames": [
{
"function": "+[RNSentry ]",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
{
"function": "+[SentryClient ]",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
{
"function": "kscrash_foobar",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
{
"function": "kscm_foobar",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
{
"function": "+[KSCrash ]",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
{
"function": "+[KSCrash]",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
{
"function": "+[KSCrashy]",
"package":
"/var/containers/Bundle/Application/B33C37A8-F933-4B6B-9FFA-152282BFDF13/SentryTest.app/SentryTest",
"instruction_addr": 4295098388,
},
]
},
"type": "NSRangeException",
}]
},
"contexts": {
"os": {
"version": "9.3.2",
"type": "os",
"name": "iOS"
}
}
}
config = load_grouping_config(get_default_grouping_config_dict())
normalize_stacktraces_for_grouping(data, grouping_config=config)
frames = data['exception']['values'][0]['stacktrace']['frames']
assert frames[0]['in_app'] is False
assert frames[1]['in_app'] is False
assert frames[2]['in_app'] is False
assert frames[3]['in_app'] is False
assert frames[4]['in_app'] is False
assert frames[5]['in_app'] is True
assert frames[6]['in_app'] is True