def spawn_capture_nodestore_stats(cache_key, project_id, event_id):
if not _should_capture_nodestore_stats(event_id):
event_processing_store.delete_by_key(_get_unprocessed_key(cache_key))
return
from sentry.tasks.reprocessing2 import capture_nodestore_stats
capture_nodestore_stats.delay(cache_key=cache_key,
project_id=project_id,
event_id=event_id)
def capture_nodestore_stats(cache_key, project_id, event_id):
set_current_project(project_id)
from sentry.eventstore.compressor import deduplicate
from sentry.eventstore.models import Event
node_id = Event.generate_node_id(project_id, event_id)
data = nodestore.get(node_id)
if not data:
metrics.incr("eventstore.compressor.error", tags={"reason": "no_data"})
return
old_event_size = _json_size(data)
unprocessed_data = event_processing_store.get(
_get_unprocessed_key(cache_key))
event_processing_store.delete_by_key(_get_unprocessed_key(cache_key))
tags = {
"with_reprocessing": bool(unprocessed_data),
"platform": data.get("platform") or "none",
"is_minidump": is_minidump_event(data),
}
if unprocessed_data:
metrics.incr("nodestore_stats.with_reprocessing")
concatenated_size = _json_size(data, unprocessed_data)
metrics.timing("events.size.concatenated",
concatenated_size,
tags=tags)
metrics.timing("events.size.concatenated.ratio",
concatenated_size / old_event_size,
tags=tags)
_data = dict(data)
_data["__nodestore_reprocessing"] = unprocessed_data
simple_concatenated_size = _json_size(_data)
metrics.timing("events.size.simple_concatenated",
simple_concatenated_size,
tags=tags)
metrics.timing(
"events.size.simple_concatenated.ratio",
simple_concatenated_size / old_event_size,
tags=tags,
)
else:
metrics.incr("nodestore_stats.without_reprocessing")
new_data, extra_keys = deduplicate(dict(data))
total_size = event_size = _json_size(new_data)
for key, value in six.iteritems(extra_keys):
if nodestore.get(key) is not None:
metrics.incr("eventstore.compressor.hits", tags=tags)
# do not continue, nodestore.set() should bump TTL
else:
metrics.incr("eventstore.compressor.misses", tags=tags)
total_size += _json_size(value)
# key is md5sum of content
# do not store actual value to keep prod impact to a minimum
nodestore.set(key, {})
metrics.timing("events.size.deduplicated", event_size, tags=tags)
metrics.timing("events.size.deduplicated.total_written",
total_size,
tags=tags)
metrics.timing("events.size.deduplicated.ratio",
event_size / old_event_size,
tags=tags)
metrics.timing("events.size.deduplicated.total_written.ratio",
total_size / old_event_size,
tags=tags)
if total_size > old_event_size:
nodestore_stats_logger.info(
"events.size.deduplicated.details",
extra={
"project_id": project_id,
"event_id": event_id,
"total_size": total_size,
"old_event_size": old_event_size,
},
)
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
set_current_project(project_id)
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer(
"tasks.store.do_save_event.get_cache") as metric_tags:
data = event_processing_store.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get(
"type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
set_current_project(project_id)
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
manager.save(project_id,
assume_normalized=True,
start_time=start_time,
cache_key=cache_key)
# Put the updated event back into the cache so that post_process
# has the most recent data.
data = manager.get_data()
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
with metrics.timer(
"tasks.store.do_save_event.write_processing_cache"):
event_processing_store.store(data)
except HashDiscarded:
# Delete the event payload from cache since it won't show up in post-processing.
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
event_processing_store.delete_by_key(cache_key)
event_processing_store.delete_by_key(
_get_unprocessed_key(cache_key))
finally:
reprocessing2.mark_event_reprocessed(data)
if cache_key:
with metrics.timer(
"tasks.store.do_save_event.delete_attachment_cache"):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])
time_synthetic_monitoring_event(data, project_id, start_time)