def test_with_attachments(default_project, task_runner, monkeypatch,
preprocess_event):
payload = get_normalized_event({"message": "hello world"}, default_project)
event_id = payload["event_id"]
attachment_id = "ca90fb45-6dd9-40a0-a18f-8693aa621abb"
project_id = default_project.id
start_time = time.time() - 3600
process_attachment_chunk({
"payload": b"Hello ",
"event_id": event_id,
"project_id": project_id,
"id": attachment_id,
"chunk_index": 0,
})
process_attachment_chunk({
"payload": b"World!",
"event_id": event_id,
"project_id": project_id,
"id": attachment_id,
"chunk_index": 1,
})
process_event({
"payload":
json.dumps(payload),
"start_time":
start_time,
"event_id":
event_id,
"project_id":
project_id,
"remote_addr":
"127.0.0.1",
"attachments": [{
"id": attachment_id,
"name": "lol.txt",
"content_type": "text/plain",
"attachment_type": "custom.attachment",
"chunks": 2,
}],
})
kwargs, = preprocess_event
cache_key = u"e:{event_id}:{project_id}".format(event_id=event_id,
project_id=project_id)
assert kwargs == {
"cache_key": cache_key,
"data": payload,
"event_id": event_id,
"project": default_project,
"start_time": start_time,
}
att, = attachment_cache.get(cache_key)
assert att.data == b"Hello World!"
assert att.name == "lol.txt"
assert att.content_type == "text/plain"
assert att.type == "custom.attachment"
def reprocess_minidump_with_cfi(data):
"""Reprocesses a minidump event if CFI(call frame information) is available
and viable. The event is only processed if there are stack traces that
contain scanned frames.
"""
handle = ThreadProcessingHandle(data)
# Check stacktrace caches first and skip all that do not need CFI. This is
# either if a thread is trusted (i.e. it does not contain scanned frames) or
# since it can be fetched from the cache.
threads = {}
for tid, thread in handle.iter_threads():
if not thread.needs_cfi:
continue
if thread.load_from_cache():
if thread.apply_to_event():
handle.indicate_change()
continue
threads[tid] = thread
if not threads:
return handle.result()
# Check if we have a minidump to reprocess
cache_key = cache_key_for_event(data)
attachments = attachment_cache.get(cache_key) or []
minidump = next((a for a in attachments if a.type == MINIDUMP_ATTACHMENT_TYPE), None)
if not minidump:
return handle.result()
# Determine modules loaded into the process during the crash
debug_ids = [module.debug_id for module in handle.iter_modules()]
if not debug_ids:
return handle.result()
# Load CFI caches for all loaded modules (even unreferenced ones)
project = Project.objects.get_from_cache(id=data['project'])
cficaches = ProjectDebugFile.difcache.get_cficaches(project, debug_ids)
if not cficaches:
return handle.result()
# Reprocess the minidump with CFI
cfi_map = FrameInfoMap.new()
for debug_id, cficache in six.iteritems(cficaches):
cfi_map.add(debug_id, cficache)
state = process_minidump(minidump.data, cfi=cfi_map)
# Merge existing stack traces with new ones from the minidump
for minidump_thread in state.threads():
thread = threads.get(minidump_thread.thread_id)
if thread:
thread.load_from_minidump(minidump_thread)
thread.save_to_cache()
if thread.apply_to_event():
handle.indicate_change()
return handle.result()
def reprocess_minidump_with_cfi(data):
"""Reprocesses a minidump event if CFI(call frame information) is available
and viable. The event is only processed if there are stack traces that
contain scanned frames.
"""
handle = ThreadProcessingHandle(data)
# Check stacktrace caches first and skip all that do not need CFI. This is
# either if a thread is trusted (i.e. it does not contain scanned frames) or
# since it can be fetched from the cache.
threads = {}
for tid, thread in handle.iter_threads():
if not thread.needs_cfi:
continue
if thread.load_from_cache():
if thread.apply_to_event():
handle.indicate_change()
continue
threads[tid] = thread
if not threads:
return handle.result()
# Check if we have a minidump to reprocess
cache_key = cache_key_for_event(data)
attachments = attachment_cache.get(cache_key) or []
minidump = next((a for a in attachments if a.type == MINIDUMP_ATTACHMENT_TYPE), None)
if not minidump:
return handle.result()
# Determine modules loaded into the process during the crash
debug_ids = [module.id for module in handle.iter_modules()]
if not debug_ids:
return handle.result()
# Load CFI caches for all loaded modules (even unreferenced ones)
project = Project.objects.get_from_cache(id=data['project'])
cficaches = ProjectDebugFile.difcache.get_cficaches(project, debug_ids)
if not cficaches:
return handle.result()
# Reprocess the minidump with CFI
cfi_map = FrameInfoMap.new()
for debug_id, cficache in six.iteritems(cficaches):
cfi_map.add(debug_id, cficache)
state = process_minidump(minidump.data, cfi=cfi_map)
# Merge existing stack traces with new ones from the minidump
for minidump_thread in state.threads():
thread = threads.get(minidump_thread.thread_id)
if thread:
thread.load_from_minidump(minidump_thread)
thread.save_to_cache()
if thread.apply_to_event():
handle.indicate_change()
return handle.result()
def event_preprocessor(data):
extra = data.setdefault("extra", {})
extra.setdefault("processing_counter", 0)
extra["processing_counter"] += 1
cache_key = cache_key_for_event(data)
attachments = attachment_cache.get(cache_key)
extra.setdefault("attachments", []).append([attachment.type for attachment in attachments])
return data
def get_attachments(cache_key, job):
"""
Retrieves the list of attachments for this event.
This method skips attachments that have been marked for rate limiting by
earlier ingestion pipeline.
:param cache_key: The cache key at which the event payload is stored in the
cache. This is used to retrieve attachments.
:param job: The job context container.
"""
if cache_key is None:
return []
project = job["event"].project
if not features.has("organizations:event-attachments", project.organization, actor=None):
return []
attachments = list(attachment_cache.get(cache_key))
if not attachments:
return []
return [attachment for attachment in attachments if not attachment.rate_limited]
def get_event_attachment(data, attachment_type):
cache_key = cache_key_for_event(data)
attachments = attachment_cache.get(cache_key) or []
return next((a for a in attachments if a.type == attachment_type), None)
def get_attachments(cache_key, event):
"""
Computes a list of attachments that should be stored.
This method checks whether event attachments are available and sends them to
the blob store. There is special handling for crash reports which may
contain unstripped PII. If the project or organization is configured to
limit the amount of crash reports per group, the number of stored crashes is
limited.
:param cache_key: The cache key at which the event payload is stored in the
cache. This is used to retrieve attachments.
:param event: The event model instance.
"""
filtered = []
if cache_key is None:
return filtered
project = event.project
if not features.has("organizations:event-attachments",
project.organization,
actor=None):
return filtered
attachments = list(attachment_cache.get(cache_key))
if not attachments:
return filtered
# The setting is both an organization and project setting. The project
# setting strictly overrides the organization setting, unless set to the
# default.
max_crashreports = get_max_crashreports(project)
if not max_crashreports:
max_crashreports = get_max_crashreports(project.organization)
# The number of crash reports is cached per group
crashreports_key = get_crashreport_key(event.group_id)
# Only fetch the number of stored crash reports if there is a crash report
# in the list of attachments. Otherwise, we won't require this number.
if any(attachment.type in CRASH_REPORT_TYPES
for attachment in attachments):
cached_reports = get_stored_crashreports(crashreports_key, event,
max_crashreports)
else:
cached_reports = 0
stored_reports = cached_reports
for attachment in attachments:
# If the attachment is a crash report (e.g. minidump), we need to honor
# the store_crash_reports setting. Otherwise, we assume that the client
# has already verified PII and just store the attachment.
if attachment.type in CRASH_REPORT_TYPES:
if crashreports_exceeded(stored_reports, max_crashreports):
continue
stored_reports += 1
filtered.append(attachment)
# Check if we have exceeded the stored crash reports count. If so, we
# persist the current maximum (not the actual number!) into the cache. Next
# time when loading from the cache, we will validate that this number has
# not changed, or otherwise re-fetch from the database.
if crashreports_exceeded(
stored_reports,
max_crashreports) and stored_reports > cached_reports:
cache.set(crashreports_key, max_crashreports, CRASH_REPORT_TIMEOUT)
return filtered
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager, track_outcome
from sentry import quotas
from sentry.models import ProjectKey
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
key_id = None if data is None else data.get('key_id')
timestamp = to_datetime(start_time) if start_time is not None else None
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments',
event.project.organization,
actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(event.project.organization_id, event.project.id, key_id,
'accepted', None, timestamp)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
track_outcome(project.organization_id, project_id, key_id, 'filtered',
reason, timestamp)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or \
features.has('organizations:event-attachments', event.project.organization, actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
Raven.tags_context({
'project': project_id,
})
try:
manager = EventManager(data)
event = manager.save(project_id)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments',
event.project.organization,
actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
except HashDiscarded:
increment_list = [
(tsdb.models.project_total_received_discarded, project_id),
]
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
increment_list.extend([
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_blacklisted,
project.organization_id),
])
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(
id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
else:
increment_list.append(
(tsdb.models.key_total_blacklisted, project_key.id))
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
tsdb.incr_multi(
increment_list,
timestamp=to_datetime(start_time)
if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
attachment_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
key_id = None if data is None else data.get('key_id')
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'post'},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments', event.project.organization, actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id
)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or \
features.has('organizations:event-attachments', event.project.organization, actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
from sentry.ingest.outcomes_consumer import mark_signal_sent
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
key_id = None if data is None else data.get("key_id")
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
# event.project.organization is populated after this statement.
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has("organizations:event-attachments",
event.project.organization,
actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id,
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
# There is no signal supposed to be sent for this particular
# outcome-reason combination. Prevent the outcome consumer from
# emitting it for now.
#
# XXX(markus): Revisit decision about signals once outcomes consumer is stable.
mark_signal_sent(event_id, project_id)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id,
)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or features.has("organizations:event-attachments",
event.project.organization,
actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])
def get_attached_minidump(data):
cache_key = cache_key_for_event(data)
attachments = attachment_cache.get(cache_key) or []
return next((a for a in attachments if a.type == MINIDUMP_ATTACHMENT_TYPE),
None)
def get_attached_minidump(data):
cache_key = cache_key_for_event(data)
attachments = attachment_cache.get(cache_key) or []
return next((a for a in attachments if a.type == MINIDUMP_ATTACHMENT_TYPE), None)
def save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
Raven.tags_context({
'project': project_id,
})
try:
manager = EventManager(data)
event = manager.save(project_id)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments', event.project.organization, actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
except HashDiscarded:
increment_list = [
(tsdb.models.project_total_received_discarded, project_id),
]
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
increment_list.extend([
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_blacklisted, project.organization_id),
])
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
else:
increment_list.append((tsdb.models.key_total_blacklisted, project_key.id))
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
tsdb.incr_multi(
increment_list,
timestamp=to_datetime(start_time) if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
