def write_applecrashreport_placeholder(data):
"""
Writes a placeholder to indicate that this event has an apple crash report.
This will indicate to the ingestion pipeline that this event will need to be
processed. The payload can be checked via ``is_applecrashreport_event``.
"""
# Apple crash report events must be native platform for processing.
data["platform"] = "native"
# Assume that this minidump is the result of a crash and assign the fatal
# level. Note that the use of `setdefault` here doesn't generally allow the
# user to override the minidump's level as processing will overwrite it
# later.
setdefault_path(data, "level", value="fatal")
# Create a placeholder exception. This signals normalization that this is an
# error event and also serves as a placeholder if processing of the minidump
# fails.
exception = {
"type": "AppleCrashReport",
"value": "Invalid Apple Crash Report",
"mechanism": {
"type": "applecrashreport",
"handled": False,
"synthetic": True
},
}
data["exception"] = {"values": [exception]}
def set_in_app(frame, value):
orig_in_app = frame.get("in_app")
if orig_in_app == value:
return
orig_in_app = int(orig_in_app) if orig_in_app is not None else -1
setdefault_path(frame, "data", "orig_in_app", value=orig_in_app)
frame["in_app"] = value
def test_set_default(self):
data = {'a': {'b': 2}}
assert not setdefault_path(data, 'a', 'b', value=42)
assert data == {'a': {'b': 2}}
data = {}
assert setdefault_path(data, 'a', 'b', value=42)
assert data == {'a': {'b': 42}}
def test_set_default(self):
data = {"a": {"b": 2}}
assert not setdefault_path(data, "a", "b", value=42)
assert data == {"a": {"b": 2}}
data = {}
assert setdefault_path(data, "a", "b", value=42)
assert data == {"a": {"b": 42}}
def set_in_app(frame, value):
orig_in_app = frame.get('in_app')
if orig_in_app == value:
return
orig_in_app = int(orig_in_app) if orig_in_app is not None else -1
setdefault_path(frame, 'data', 'orig_in_app', value=orig_in_app)
frame['in_app'] = value
def test_set_default(self):
data = {'a': {'b': 2}}
assert not setdefault_path(data, 'a', 'b', value=42)
assert data == {'a': {'b': 2}}
data = {}
assert setdefault_path(data, 'a', 'b', value=42)
assert data == {'a': {'b': 42}}
def merge_unreal_logs_event(unreal_logs, event):
setdefault_path(event, 'breadcrumbs', 'values', value=[])
breadcrumbs = event['breadcrumbs']['values']
for log in unreal_logs:
breadcrumbs.append({
'timestamp': log.get('timestamp'),
'category': log.get('component'),
'message': log.get('message'),
})
def normalize_user_agent(data):
user_agent = _parse_user_agent(data)
if not user_agent:
return
setdefault_path(data, 'contexts', value={})
_inject_browser_context(data, user_agent)
_inject_os_context(data, user_agent)
_inject_device_context(data, user_agent)
def normalize_user_agent(data):
user_agent = _parse_user_agent(data)
if not user_agent:
return
setdefault_path(data, 'contexts', value={})
_inject_browser_context(data, user_agent)
_inject_os_context(data, user_agent)
_inject_device_context(data, user_agent)
def merge_unreal_logs_event(unreal_logs, event):
setdefault_path(event, 'breadcrumbs', 'values', value=[])
breadcrumbs = event['breadcrumbs']['values']
for log in unreal_logs:
breadcrumbs.append({
'timestamp': log.get('timestamp'),
'category': log.get('component'),
'message': log.get('message'),
})
def _inject_os_context(data, user_agent):
ua = user_agent['os']
try:
if ua['family'] == 'Other':
return
setdefault_path(data, 'contexts', 'os', value={
'name': ua['family'],
'version': _get_version(ua),
})
except KeyError:
pass
def _inject_os_context(data, user_agent):
ua = user_agent['os']
try:
if ua['family'] == 'Other':
return
setdefault_path(data, 'contexts', 'os', value={
'name': ua['family'],
'version': _get_version(ua),
})
except KeyError:
pass
def merge_unreal_logs_event(unreal_logs, event):
setdefault_path(event, "breadcrumbs", "values", value=[])
breadcrumbs = event["breadcrumbs"]["values"]
for log in unreal_logs:
message = log.get("message")
if message:
breadcrumbs.append({
"timestamp": log.get("timestamp"),
"category": log.get("component"),
"message": message,
})
def _inject_device_context(data, user_agent):
ua = user_agent['device']
try:
if ua['family'] == 'Other':
return
setdefault_path(data, 'contexts', 'device', value={
'family': ua['family'],
'model': ua['model'],
'brand': ua['brand'],
})
except KeyError:
pass
def _inject_device_context(data, user_agent):
ua = user_agent['device']
try:
if ua['family'] == 'Other':
return
setdefault_path(data, 'contexts', 'device', value={
'family': ua['family'],
'model': ua['model'],
'brand': ua['brand'],
})
except KeyError:
pass
def _pull_out_data(jobs, projects):
"""
A bunch of (probably) CPU bound stuff.
"""
for job in jobs:
job["project_id"] = int(job["project_id"])
data = job["data"]
# Pull the toplevel data we're interested in
job["culprit"] = get_culprit(data)
transaction_name = data.get("transaction")
if transaction_name:
transaction_name = force_text(transaction_name)
job["transaction"] = transaction_name
key_id = None if data is None else data.get("key_id")
if key_id is not None:
key_id = int(key_id)
job["key_id"] = key_id
job["logger_name"] = logger_name = data.get("logger")
job["level"] = level = data.get("level")
job["release"] = data.get("release")
job["dist"] = data.get("dist")
job["environment"] = environment = data.get("environment")
job["recorded_timestamp"] = data.get("timestamp")
job["event"] = event = _get_event_instance(job["data"], project_id=job["project_id"])
job["data"] = data = event.data.data
job["category"] = DataCategory.from_event_type(data.get("type"))
job["platform"] = event.platform
event._project_cache = projects[job["project_id"]]
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, "tags", value=[])
set_tag(data, "level", level)
if logger_name:
set_tag(data, "logger", logger_name)
if environment:
set_tag(data, "environment", environment)
if transaction_name:
set_tag(data, "transaction", transaction_name)
job["received_timestamp"] = job["event"].data.get("received") or float(
job["event"].datetime.strftime("%s")
)
def merge_symbolicator_minidump_system_info(data, system_info):
set_path(data, 'contexts', 'os', 'type',
value='os') # Required by "get_sdk_from_event"
setdefault_path(data,
'contexts',
'os',
'name',
value=system_info.get('os_name'))
setdefault_path(data,
'contexts',
'os',
'version',
value=system_info.get('os_version'))
setdefault_path(data,
'contexts',
'os',
'build',
value=system_info.get('os_build'))
set_path(data, 'contexts', 'device', 'type', value='device')
setdefault_path(data,
'contexts',
'device',
'arch',
value=system_info.get('cpu_arch'))
def _merge_system_info(data, system_info):
set_path(data, "contexts", "os", "type",
value="os") # Required by "get_sdk_from_event"
setdefault_path(data,
"contexts",
"os",
"name",
value=system_info.get("os_name"))
setdefault_path(data,
"contexts",
"os",
"version",
value=system_info.get("os_version"))
setdefault_path(data,
"contexts",
"os",
"build",
value=system_info.get("os_build"))
set_path(data, "contexts", "device", "type", value="device")
setdefault_path(data,
"contexts",
"device",
"arch",
value=system_info.get("cpu_arch"))
def write_minidump_placeholder(data):
# Minidump events must be native platform.
data["platform"] = "native"
# Assume that this minidump is the result of a crash and assign the fatal
# level. Note that the use of `setdefault` here doesn't generally allow the
# user to override the minidump's level as processing will overwrite it
# later.
setdefault_path(data, "level", value="fatal")
# Create a placeholder exception. This signals normalization that this is an
# error event and also serves as a placeholder if processing of the minidump
# fails.
exception = {
"type": "Minidump",
"value": "Invalid Minidump",
"mechanism": {"type": "minidump", "handled": False, "synthetic": True},
}
data["exception"] = {"values": [exception]}
def write_minidump_placeholder(data):
# Minidump events must be native platform.
data['platform'] = 'native'
# Assume that this minidump is the result of a crash and assign the fatal
# level. Note that the use of `setdefault` here doesn't generally allow the
# user to override the minidump's level as processing will overwrite it
# later.
setdefault_path(data, 'level', value='fatal')
# Create a placeholder exception. This signals normalization that this is an
# error event and also serves as a placeholder if processing of the minidump
# fails.
exception = {
'type': 'Minidump',
'value': 'Invalid Minidump',
'mechanism': {
'type': 'minidump',
'handled': False,
'synthetic': True,
}
}
data['exception'] = {'values': [exception]}
def save(self, project_id, raw=False, assume_normalized=False):
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
# Make sure we cache on the project before returning
event._project_cache = project
logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
}
)
return event
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get('level')
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get('transaction')
logger_name = data.get('logger')
release = data.get('release')
dist = data.get('dist')
environment = data.get('environment')
recorded_timestamp = data.get('timestamp')
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, 'tags', value=[])
set_tag(data, 'level', level)
if logger_name:
set_tag(data, 'logger', logger_name)
if environment:
set_tag(data, 'environment', environment)
if transaction_name:
set_tag(data, 'transaction', transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, 'release')
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
set_tag(data, 'sentry:release', release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, 'dist')
set_tag(data, 'sentry:dist', dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, 'user')
set_tag(data, 'sentry:user', event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project))
normalize_stacktraces_for_grouping(data, grouping_config)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right paramters.
data['fingerprint'] = data.get('fingerprint') or ['{{ default }}']
apply_server_fingerprinting(data, get_fingerprinting_config_for_project(project))
# Here we try to use the grouping config that was requested in the
# event. If that config has since been deleted (because it was an
# experimental grouping config) we fall back to the default.
try:
hashes = event.get_hashes()
except GroupingConfigNotFound:
data['grouping_config'] = get_grouping_config_dict_for_project(project)
hashes = event.get_hashes()
data['hashes'] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overriden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
event_metadata = materialized_metadata['metadata']
data.update(materialized_metadata)
data['culprit'] = culprit
# index components into ``Event.message``
# See GH-3248
event.message = self.get_search_message(event_metadata, culprit)
received_timestamp = event.data.get('received') or float(event.datetime.strftime('%s'))
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata['last_received'] = received_timestamp
kwargs = {
'platform': platform,
'message': event.message,
'culprit': culprit,
'logger': logger_name,
'level': LOG_LEVELS_MAP.get(level),
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': group_metadata,
}
if release:
kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs
)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project, group=group, event_id=event_id)
except IntegrityError:
logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
}
)
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release': release if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters, timestamp=event.datetime, environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append(
(tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
})
)
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# Update any event attachment that arrived before the event group was defined.
EventAttachment.objects.filter(
project_id=project.id,
event_id=event_id,
).update(
group_id=group.id,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
}
)
return event
tagstore.delay_index_event_tags(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=event.tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id, (event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id, (event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(
ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
}
)
if is_new_group_environment:
buffer.incr(
ReleaseProjectEnvironment, {'new_issues_count': 1}, {
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
}
)
safe_execute(
Group.objects.add_tags,
group,
environment,
event.get_tags(),
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project, group=group, sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
metrics.timing(
'events.size.data.post_save',
event.size,
tags={'project_id': project.id}
)
return event
def _merge_system_info(data, system_info):
set_path(data, "contexts", "os", "type",
value="os") # Required by "get_sdk_from_event"
os_name = system_info.get("os_name")
os_version = system_info.get("os_version")
os_build = system_info.get("os_build")
if os_version:
setdefault_path(data, "contexts", "os", "version", value=os_version)
if os_build:
setdefault_path(data, "contexts", "os", "build", value=os_build)
if os_name and not os_version and not os_build:
setdefault_path(data,
"contexts",
"os",
"raw_description",
value=os_name)
elif os_name:
setdefault_path(data, "contexts", "os", "name", value=os_name)
set_path(data, "contexts", "device", "type", value="device")
setdefault_path(data,
"contexts",
"device",
"arch",
value=system_info.get("cpu_arch"))
device_model = system_info.get("device_model")
if device_model:
setdefault_path(data,
"contexts",
"device",
"model",
value=device_model)
def save(self,
project_id,
raw=False,
assume_normalized=False,
cache_key=None):
"""
We re-insert events with duplicate IDs into Snuba, which is responsible
for deduplicating events. Since deduplication in Snuba is on the primary
key (based on event ID, project ID and day), events with same IDs are only
deduplicated if their timestamps fall on the same day. The latest event
always wins and overwrites the value of events received earlier in that day.
Since we increment counters and frequencies here before events get inserted
to eventstream these numbers may be larger than the total number of
events if we receive duplicate event IDs that fall on the same day
(that do not hit cache first).
"""
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get("level")
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get("transaction")
logger_name = data.get("logger")
release = data.get("release")
dist = data.get("dist")
environment = data.get("environment")
recorded_timestamp = data.get("timestamp")
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Right now the event type is the signal to skip the group. This
# is going to change a lot.
if event.get_event_type() == "transaction":
issueless_event = True
else:
issueless_event = False
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, "tags", value=[])
set_tag(data, "level", level)
if logger_name:
set_tag(data, "logger", logger_name)
if environment:
set_tag(data, "environment", environment)
if transaction_name:
set_tag(data, "transaction", transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, "release")
release = Release.get_or_create(project=project,
version=release,
date_added=date)
set_tag(data, "sentry:release", release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, "dist")
set_tag(data, "sentry:dist", dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, "user")
set_tag(data, "sentry:user", event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project))
normalize_stacktraces_for_grouping(data, grouping_config)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags,
event,
_with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right parameters.
data["fingerprint"] = data.get("fingerprint") or ["{{ default }}"]
apply_server_fingerprinting(
data, get_fingerprinting_config_for_project(project))
# Here we try to use the grouping config that was requested in the
# event. If that config has since been deleted (because it was an
# experimental grouping config) we fall back to the default.
try:
hashes = event.get_hashes()
except GroupingConfigNotFound:
data["grouping_config"] = get_grouping_config_dict_for_project(
project)
hashes = event.get_hashes()
data["hashes"] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overridden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
data.update(materialized_metadata)
data["culprit"] = culprit
received_timestamp = event.data.get("received") or float(
event.datetime.strftime("%s"))
if not issueless_event:
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata["last_received"] = received_timestamp
kwargs = {
"platform": platform,
"message": event.search_message,
"culprit": culprit,
"logger": logger_name,
"level": LOG_LEVELS_MAP.get(level),
"last_seen": date,
"first_seen": date,
"active_at": date,
"data": group_metadata,
}
if release:
kwargs["first_release"] = release
try:
group, is_new, is_regression = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs)
except HashDiscarded:
event_discarded.send_robust(project=project,
sender=EventManager)
metrics.incr(
"events.discarded",
skip_internal=True,
tags={
"organization_id": project.organization_id,
"platform": platform
},
)
raise
else:
event_saved.send_robust(project=project,
event_size=event.size,
sender=EventManager)
event.group = group
else:
group = None
is_new = False
is_regression = False
event_saved.send_robust(project=project,
event_size=event.size,
sender=EventManager)
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
environment = Environment.get_or_create(project=project,
name=environment)
if group:
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={"first_release": release if release else None},
)
else:
is_new_group_environment = False
if release:
ReleaseEnvironment.get_or_create(project=project,
release=release,
environment=environment,
datetime=date)
ReleaseProjectEnvironment.get_or_create(project=project,
release=release,
environment=environment,
datetime=date)
if group:
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date)
counters = [(tsdb.models.project, project.id)]
if group:
counters.append((tsdb.models.group, group.id))
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
frequencies = []
if group:
frequencies.append((tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1
}
}))
if release:
frequencies.append((tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1
}
}))
if frequencies:
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
if group:
UserReport.objects.filter(project=project,
event_id=event_id).update(
group=group, environment=environment)
# Enusre the _metrics key exists. This is usually created during
# and prefilled with ingestion sizes.
event_metrics = event.data.get("_metrics") or {}
event.data["_metrics"] = event_metrics
# Capture the actual size that goes into node store.
event_metrics["bytes.stored.event"] = len(
json.dumps(dict(event.data.items())))
# Load attachments first, but persist them at the very last after
# posting to eventstream to make sure all counters and eventstream are
# incremented for sure.
attachments = self.get_attachments(cache_key, event)
for attachment in attachments:
key = "bytes.stored.%s" % (attachment.type, )
event_metrics[key] = (event_metrics.get(key) or 0) + len(
attachment.data)
# Write the event to Nodestore
event.data.save()
if event_user:
counters = [(tsdb.models.users_affected_by_project, project.id,
(event_user.tag_value, ))]
if group:
counters.append((tsdb.models.users_affected_by_group, group.id,
(event_user.tag_value, )))
tsdb.record_multi(counters,
timestamp=event.datetime,
environment_id=environment.id)
if release:
if is_new:
buffer.incr(
ReleaseProject,
{"new_groups": 1},
{
"release_id": release.id,
"project_id": project.id
},
)
if is_new_group_environment:
buffer.incr(
ReleaseProjectEnvironment,
{"new_issues_count": 1},
{
"project_id": project.id,
"release_id": release.id,
"environment_id": environment.id,
},
)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project,
event=event,
sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
# Do this last to ensure signals get emitted even if connection to the
# file store breaks temporarily.
self.save_attachments(attachments, event)
metric_tags = {"from_relay": "_relay_processed" in self._data}
metrics.timing("events.latency",
received_timestamp - recorded_timestamp,
tags=metric_tags)
metrics.timing("events.size.data.post_save",
event.size,
tags=metric_tags)
metrics.incr(
"events.post_save.normalize.errors",
amount=len(self._data.get("errors") or ()),
tags=metric_tags,
)
return event
def save(self, project_id, raw=False, assume_normalized=False):
# Normalize if needed
if not self._normalized:
if not assume_normalized:
self.normalize()
self._normalized = True
data = self._data
project = Project.objects.get_from_cache(id=project_id)
project._organization_cache = Organization.objects.get_from_cache(
id=project.organization_id)
# Check to make sure we're not about to do a bunch of work that's
# already been done if we've processed an event with this ID. (This
# isn't a perfect solution -- this doesn't handle ``EventMapping`` and
# there's a race condition between here and when the event is actually
# saved, but it's an improvement. See GH-7677.)
try:
event = Event.objects.get(
project_id=project.id,
event_id=data['event_id'],
)
except Event.DoesNotExist:
pass
else:
# Make sure we cache on the project before returning
event._project_cache = project
logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': data['event_id'],
'project_id': project.id,
'model': Event.__name__,
}
)
return event
# Pull out the culprit
culprit = self.get_culprit()
# Pull the toplevel data we're interested in
level = data.get('level')
# TODO(mitsuhiko): this code path should be gone by July 2018.
# This is going to be fine because no code actually still depends
# on integers here. When we need an integer it will be converted
# into one later. Old workers used to send integers here.
if level is not None and isinstance(level, six.integer_types):
level = LOG_LEVELS[level]
transaction_name = data.get('transaction')
logger_name = data.get('logger')
release = data.get('release')
dist = data.get('dist')
environment = data.get('environment')
recorded_timestamp = data.get('timestamp')
# We need to swap out the data with the one internal to the newly
# created event object
event = self._get_event_instance(project_id=project_id)
self._data = data = event.data.data
event._project_cache = project
date = event.datetime
platform = event.platform
event_id = event.event_id
if transaction_name:
transaction_name = force_text(transaction_name)
# Some of the data that are toplevel attributes are duplicated
# into tags (logger, level, environment, transaction). These are
# different from legacy attributes which are normalized into tags
# ahead of time (site, server_name).
setdefault_path(data, 'tags', value=[])
set_tag(data, 'level', level)
if logger_name:
set_tag(data, 'logger', logger_name)
if environment:
set_tag(data, 'environment', environment)
if transaction_name:
set_tag(data, 'transaction', transaction_name)
if release:
# dont allow a conflicting 'release' tag
pop_tag(data, 'release')
release = Release.get_or_create(
project=project,
version=release,
date_added=date,
)
set_tag(data, 'sentry:release', release.version)
if dist and release:
dist = release.add_dist(dist, date)
# dont allow a conflicting 'dist' tag
pop_tag(data, 'dist')
set_tag(data, 'sentry:dist', dist.name)
else:
dist = None
event_user = self._get_event_user(project, data)
if event_user:
# dont allow a conflicting 'user' tag
pop_tag(data, 'user')
set_tag(data, 'sentry:user', event_user.tag_value)
# At this point we want to normalize the in_app values in case the
# clients did not set this appropriately so far.
grouping_config = load_grouping_config(
get_grouping_config_dict_for_event_data(data, project))
normalize_stacktraces_for_grouping(data, grouping_config)
for plugin in plugins.for_project(project, version=None):
added_tags = safe_execute(plugin.get_tags, event, _with_transaction=False)
if added_tags:
# plugins should not override user provided tags
for key, value in added_tags:
if get_tag(data, key) is None:
set_tag(data, key, value)
for path, iface in six.iteritems(event.interfaces):
for k, v in iface.iter_tags():
set_tag(data, k, v)
# Get rid of ephemeral interface data
if iface.ephemeral:
data.pop(iface.path, None)
# The active grouping config was put into the event in the
# normalize step before. We now also make sure that the
# fingerprint was set to `'{{ default }}' just in case someone
# removed it from the payload. The call to get_hashes will then
# look at `grouping_config` to pick the right paramters.
data['fingerprint'] = data.get('fingerprint') or ['{{ default }}']
apply_server_fingerprinting(data, get_fingerprinting_config_for_project(project))
hashes = event.get_hashes()
data['hashes'] = hashes
# we want to freeze not just the metadata and type in but also the
# derived attributes. The reason for this is that we push this
# data into kafka for snuba processing and our postprocessing
# picks up the data right from the snuba topic. For most usage
# however the data is dynamically overriden by Event.title and
# Event.location (See Event.as_dict)
materialized_metadata = self.materialize_metadata()
event_metadata = materialized_metadata['metadata']
data.update(materialized_metadata)
data['culprit'] = culprit
# index components into ``Event.message``
# See GH-3248
event.message = self.get_search_message(event_metadata, culprit)
received_timestamp = event.data.get('received') or float(event.datetime.strftime('%s'))
# The group gets the same metadata as the event when it's flushed but
# additionally the `last_received` key is set. This key is used by
# _save_aggregate.
group_metadata = dict(materialized_metadata)
group_metadata['last_received'] = received_timestamp
kwargs = {
'platform': platform,
'message': event.message,
'culprit': culprit,
'logger': logger_name,
'level': LOG_LEVELS_MAP.get(level),
'last_seen': date,
'first_seen': date,
'active_at': date,
'data': group_metadata,
}
if release:
kwargs['first_release'] = release
try:
group, is_new, is_regression, is_sample = self._save_aggregate(
event=event, hashes=hashes, release=release, **kwargs
)
except HashDiscarded:
event_discarded.send_robust(
project=project,
sender=EventManager,
)
metrics.incr(
'events.discarded',
skip_internal=True,
tags={
'organization_id': project.organization_id,
'platform': platform,
},
)
raise
else:
event_saved.send_robust(
project=project,
event_size=event.size,
sender=EventManager,
)
event.group = group
# store a reference to the group id to guarantee validation of isolation
event.data.bind_ref(event)
# When an event was sampled, the canonical source of truth
# is the EventMapping table since we aren't going to be writing out an actual
# Event row. Otherwise, if the Event isn't being sampled, we can safely
# rely on the Event table itself as the source of truth and ignore
# EventMapping since it's redundant information.
if is_sample:
try:
with transaction.atomic(using=router.db_for_write(EventMapping)):
EventMapping.objects.create(project=project, group=group, event_id=event_id)
except IntegrityError:
logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': EventMapping.__name__,
}
)
return event
environment = Environment.get_or_create(
project=project,
name=environment,
)
group_environment, is_new_group_environment = GroupEnvironment.get_or_create(
group_id=group.id,
environment_id=environment.id,
defaults={
'first_release': release if release else None,
},
)
if release:
ReleaseEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
ReleaseProjectEnvironment.get_or_create(
project=project,
release=release,
environment=environment,
datetime=date,
)
grouprelease = GroupRelease.get_or_create(
group=group,
release=release,
environment=environment,
datetime=date,
)
counters = [
(tsdb.models.group, group.id),
(tsdb.models.project, project.id),
]
if release:
counters.append((tsdb.models.release, release.id))
tsdb.incr_multi(counters, timestamp=event.datetime, environment_id=environment.id)
frequencies = [
# (tsdb.models.frequent_projects_by_organization, {
# project.organization_id: {
# project.id: 1,
# },
# }),
# (tsdb.models.frequent_issues_by_project, {
# project.id: {
# group.id: 1,
# },
# })
(tsdb.models.frequent_environments_by_group, {
group.id: {
environment.id: 1,
},
})
]
if release:
frequencies.append(
(tsdb.models.frequent_releases_by_group, {
group.id: {
grouprelease.id: 1,
},
})
)
tsdb.record_frequency_multi(frequencies, timestamp=event.datetime)
UserReport.objects.filter(
project=project,
event_id=event_id,
).update(
group=group,
environment=environment,
)
# save the event unless its been sampled
if not is_sample:
try:
with transaction.atomic(using=router.db_for_write(Event)):
event.save()
except IntegrityError:
logger.info(
'duplicate.found',
exc_info=True,
extra={
'event_uuid': event_id,
'project_id': project.id,
'group_id': group.id,
'model': Event.__name__,
}
)
return event
tagstore.delay_index_event_tags(
organization_id=project.organization_id,
project_id=project.id,
group_id=group.id,
environment_id=environment.id,
event_id=event.id,
tags=event.tags,
date_added=event.datetime,
)
if event_user:
tsdb.record_multi(
(
(tsdb.models.users_affected_by_group, group.id, (event_user.tag_value, )),
(tsdb.models.users_affected_by_project, project.id, (event_user.tag_value, )),
),
timestamp=event.datetime,
environment_id=environment.id,
)
if release:
if is_new:
buffer.incr(
ReleaseProject, {'new_groups': 1}, {
'release_id': release.id,
'project_id': project.id,
}
)
if is_new_group_environment:
buffer.incr(
ReleaseProjectEnvironment, {'new_issues_count': 1}, {
'project_id': project.id,
'release_id': release.id,
'environment_id': environment.id,
}
)
safe_execute(
Group.objects.add_tags,
group,
environment,
event.get_tags(),
_with_transaction=False)
if not raw:
if not project.first_event:
project.update(first_event=date)
first_event_received.send_robust(project=project, group=group, sender=Project)
eventstream.insert(
group=group,
event=event,
is_new=is_new,
is_sample=is_sample,
is_regression=is_regression,
is_new_group_environment=is_new_group_environment,
primary_hash=hashes[0],
# We are choosing to skip consuming the event back
# in the eventstream if it's flagged as raw.
# This means that we want to publish the event
# through the event stream, but we don't care
# about post processing and handling the commit.
skip_consume=raw,
)
metrics.timing(
'events.latency',
received_timestamp - recorded_timestamp,
tags={
'project_id': project.id,
},
)
metrics.timing(
'events.size.data.post_save',
event.size,
tags={'project_id': project.id}
)
return event
