def _update_cachefiles(self, project, debug_files, cls):
rv = []
conversion_errors = {}
for debug_file in debug_files:
debug_id = debug_file.debug_id
# Find all the known bad files we could not convert last time. We
# use the debug identifier and file checksum to identify the source
# DIF for historic reasons (debug_file.id would do, too).
cache_key = 'scbe:%s:%s' % (debug_id, debug_file.file.checksum)
err = default_cache.get(cache_key)
if err is not None:
conversion_errors[debug_id] = err
continue
# Download the original debug symbol and convert the object file to
# a cache. This can either yield a cache object, an error or none of
# the above. THE FILE DOWNLOAD CAN TAKE SIGNIFICANT TIME.
with debug_file.file.getfile(as_tempfile=True) as tf:
file, cache, err = self._update_cachefile(debug_file, tf.name, cls)
# Store this conversion error so that we can skip subsequent
# conversions. There might be concurrent conversions running for the
# same debug file, however.
if err is not None:
default_cache.set(cache_key, err, CONVERSION_ERROR_TTL)
conversion_errors[debug_id] = err
continue
if file is not None or cache is not None:
rv.append((debug_id, file, cache))
return rv, conversion_errors
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs):
from sentry.plugins import plugins
if cache_key:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'})
error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key})
return
project = data['project']
Raven.tags_context({
'project': project,
})
# Iterate over all plugins looking for processors based on the input data
# plugins should yield a processor function only if it actually can operate
# on the input data, otherwise it should yield nothing
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False)
for processor in (processors or ()):
# On the first processor found, we just defer to the process_event
# queue to handle the actual work.
process_event.delay(cache_key=cache_key, start_time=start_time)
return
# If we get here, that means the event had no preprocessing needed to be done
# so we can jump directly to save_event
if cache_key:
data = None
save_event.delay(cache_key=cache_key, data=data, start_time=start_time)
def get(self, request, wizard_hash=None):
"""
This tries to retrieve and return the cache content if possible
otherwise creates new cache
"""
if wizard_hash is not None:
key = '%s%s' % (SETUP_WIZARD_CACHE_KEY, wizard_hash)
wizard_data = default_cache.get(key)
if wizard_data is None:
return Response(status=404)
elif wizard_data == 'empty':
# when we just created a clean cache
return Response(status=400)
return Response(serialize(wizard_data))
else:
# This creates a new available hash url for the project wizard
rate_limited = ratelimits.is_limited(
key='rl:setup-wizard:ip:%s' % request.META['REMOTE_ADDR'],
limit=10,
)
if rate_limited:
logger.info('setup-wizard.rate-limit')
return Response(
{
'Too wizard requests',
}, status=403
)
wizard_hash = get_random_string(
64, allowed_chars='abcdefghijklmnopqrstuvwxyz012345679')
key = '%s%s' % (SETUP_WIZARD_CACHE_KEY, wizard_hash)
default_cache.set(key, 'empty', SETUP_WIZARD_CACHE_TIMEOUT)
return Response(serialize({'hash': wizard_hash}))
def create_failed_event(cache_key, project, issues, event_id):
"""If processing failed we put the original data from the cache into a
raw event.
"""
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'})
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project,
event_id=event_id,
datetime=datetime.utcfromtimestamp(
data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
def process_event(cache_key, start_time=None, **kwargs):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'process'})
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
project = data['project']
Raven.tags_context({
'project': project,
})
# TODO(dcramer): ideally we would know if data changed by default
has_changed = False
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
default_cache.set(cache_key, data, 3600)
save_event.delay(cache_key=cache_key, data=None, start_time=start_time)
def _do_preprocess_event(cache_key, data, start_time, event_id,
process_event):
if cache_key:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'})
error_logger.error('preprocess.failed.empty',
extra={'cache_key': cache_key})
return
project = data['project']
Raven.tags_context({
'project': project,
})
if should_process(data):
process_event.delay(cache_key=cache_key, start_time=start_time,
event_id=event_id)
return
# If we get here, that means the event had no preprocessing needed to be done
# so we can jump directly to save_event
if cache_key:
data = None
save_event.delay(cache_key=cache_key, data=data, start_time=start_time,
event_id=event_id)
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
project = data.pop('project')
delete_raw_event(project, event_id)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process', time() - start_time,
instance=data['platform'])
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs):
from sentry.plugins import plugins
if cache_key:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'})
logger.error('Data not available in preprocess_event (cache_key=%s)', cache_key)
return
project = data['project']
# TODO(dcramer): ideally we would know if data changed by default
has_changed = False
for plugin in plugins.all(version=2):
for processor in (safe_execute(plugin.get_event_preprocessors) or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed and cache_key:
default_cache.set(cache_key, data, 3600)
if cache_key:
data = None
save_event.delay(cache_key=cache_key, data=data, start_time=start_time)
def _update_cachefiles(self, project, debug_files):
rv = []
# Find all the known bad files we could not convert last time
# around
conversion_errors = {}
for debug_file in debug_files:
cache_key = 'scbe:%s:%s' % (debug_file.debug_id, debug_file.file.checksum)
err = default_cache.get(cache_key)
if err is not None:
conversion_errors[debug_file.debug_id] = err
for debug_file in debug_files:
dsym_id = debug_file.debug_id
if dsym_id in conversion_errors:
continue
with debug_file.file.getfile(as_tempfile=True) as tf:
symcache_file, conversion_error = self._update_cachefile(
debug_file, tf)
if symcache_file is not None:
rv.append((dsym_id, symcache_file))
elif conversion_error is not None:
conversion_errors[dsym_id] = conversion_error
return rv, conversion_errors
def _do_preprocess_event(cache_key, data, start_time, event_id, process_event):
if cache_key:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}, skip_internal=False)
error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project = data['project']
with configure_scope() as scope:
scope.set_tag("project", project)
if should_process(data):
process_event.delay(cache_key=cache_key, start_time=start_time, event_id=event_id)
return
# If we get here, that means the event had no preprocessing needed to be done
# so we can jump directly to save_event
if cache_key:
data = None
save_event.delay(
cache_key=cache_key, data=data, start_time=start_time, event_id=event_id,
project_id=project
)
def reprocess_minidump(data):
project = Project.objects.get_from_cache(id=data['project'])
minidump_is_reprocessed_cache_key = minidump_reprocessed_cache_key_for_event(data)
if default_cache.get(minidump_is_reprocessed_cache_key):
return
if not _is_symbolicator_enabled(project, data):
rv = reprocess_minidump_with_cfi(data)
default_cache.set(minidump_is_reprocessed_cache_key, True, 3600)
return rv
minidump = get_attached_minidump(data)
if not minidump:
logger.error("Missing minidump for minidump event")
return
request_id_cache_key = request_id_cache_key_for_event(data)
response = run_symbolicator(
project=project,
request_id_cache_key=request_id_cache_key,
create_task=create_minidump_task,
minidump=make_buffered_slice_reader(minidump.data, None)
)
if handle_symbolicator_response_status(data, response):
merge_symbolicator_minidump_response(data, response)
event_cache_key = cache_key_for_event(data)
default_cache.set(event_cache_key, dict(data), 3600)
default_cache.set(minidump_is_reprocessed_cache_key, True, 3600)
return data
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
reprocessing_active = ProjectOption.objects.get_value(
project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT
)
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, 'sentry:sent_failed_event_hint', False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={'reprocessing_active': reprocessing_active,
'issues': issues},
).send_notification()
ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'})
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return True
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
return True
def get_assemble_status(project, checksum):
"""For a given file it checks what the current status of the assembling is.
Returns a tuple in the form ``(status, details)`` where details is either
`None` or a string identifying an error condition or notice.
"""
cache_key = 'assemble-status:%s' % _get_idempotency_id(
project, checksum)
rv = default_cache.get(cache_key)
if rv is None:
return None, None
return tuple(rv)
def public_dsn():
project_id = settings.SENTRY_FRONTEND_PROJECT or settings.SENTRY_PROJECT
cache_key = 'dsn:%s' % (project_id, )
result = default_cache.get(cache_key)
if result is None:
key = _get_project_key(project_id)
if key:
result = key.dsn_public
else:
result = ''
default_cache.set(cache_key, result, 60)
return result
def _do_process_event(cache_key, start_time, event_id):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'process'})
error_logger.error('process.failed.empty',
extra={'cache_key': cache_key})
return
project = data['project']
Raven.tags_context({
'project': project,
})
has_changed = False
# Stacktrace based event processors. These run before anything else.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data, _with_transaction=False)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
issues = data.get('processing_issues')
if issues:
create_failed_event(cache_key, project, list(issues.values()),
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
save_event.delay(cache_key=cache_key, data=None, start_time=start_time,
event_id=event_id)
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import tsdb
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
project = data.pop('project')
delete_raw_event(project, event_id, allow_hint_clear=True)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
except HashDiscarded as exc:
# TODO(jess): remove this before it goes out to a wider audience
info_logger.info(
'discarded.hash', extra={
'project_id': project,
'description': exc.message,
}
)
tsdb.incr(tsdb.models.project_total_received_discarded, project, timestamp=start_time)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def save_event(cache_key=None, data=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if data is None:
return
project = data.pop('project')
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
def save_event(cache_key=None, data=None, start_time=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if data is None:
return
project = data.pop('project')
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process', time() - start_time)
def _do_preprocess_event(cache_key, data, start_time, event_id, process_task):
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}, skip_internal=False)
error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key})
return
original_data = data
data = CanonicalKeyDict(data)
project_id = data['project']
with configure_scope() as scope:
scope.set_tag("project", project_id)
project = Project.objects.get_from_cache(id=project_id)
if should_process(data):
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id, start_time, original_data)
return
submit_save_event(project, cache_key, event_id, start_time, original_data)
def create_failed_event(
cache_key, data, project_id, issues, event_id, start_time=None, reprocessing_rev=None
):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
# We can only create failed events for events that can potentially
# create failed events.
if not reprocessing.event_supports_reprocessing(data):
return False
reprocessing_active = ProjectOption.objects.get_value(
project_id, "sentry:reprocessing_active", REPROCESSING_DEFAULT
)
# In case there is reprocessing active but the current reprocessing
# revision is already different than when we started, we want to
# immediately retry the event. This resolves the problem when
# otherwise a concurrent change of debug symbols might leave a
# reprocessing issue stuck in the project forever.
if (
reprocessing_active
and reprocessing.get_reprocessing_revision(project_id, cached=False) != reprocessing_rev
):
raise RetryProcessing()
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, "sentry:sent_failed_event_hint", False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={"reprocessing_active": reprocessing_active, "issues": issues},
).send_notification()
ProjectOption.objects.set_value(project, "sentry:sent_failed_event_hint", True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed", tags={"reason": "cache", "stage": "raw"}, skip_internal=False)
error_logger.error("process.failed_raw.empty", extra={"cache_key": cache_key})
return True
data = CanonicalKeyDict(data)
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data["timestamp"]).replace(tzinfo=timezone.utc),
data=data,
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue["scope"],
object=issue["object"],
type=issue["type"],
data=issue["data"],
)
default_cache.delete(cache_key)
return True
def _do_process_event(cache_key, start_time, event_id, process_task):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'process'
})
error_logger.error('process.failed.empty',
extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project = data['project']
with configure_scope() as scope:
scope.set_tag("project", project)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project)
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in (enhancers or ()):
enhanced = safe_execute(enhancer, data)
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
'project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key,
project,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
process_task.delay(cache_key,
start_time=start_time,
event_id=event_id)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
default_cache.set(cache_key, data, 3600)
save_event.delay(cache_key=cache_key,
data=None,
start_time=start_time,
event_id=event_id,
project_id=project)
def get(self, request, wizard_hash):
"""
This opens a page where with an active session fill stuff into the cache
Redirects to organization whenever cache has been deleted
"""
context = {
'hash': wizard_hash
}
key = '%s%s' % (SETUP_WIZARD_CACHE_KEY, wizard_hash)
wizard_data = default_cache.get(key)
if wizard_data is None:
return self.redirect_to_org(request)
orgs = Organization.objects.filter(
member_set__role__in=[x.id for x in roles.with_scope('org:read')],
member_set__user=request.user,
status=OrganizationStatus.VISIBLE,
).order_by('-date_added')[:50]
filled_projects = []
for org in orgs:
projects = list(Project.objects.filter(
organization=org,
status=ProjectStatus.VISIBLE,
).order_by('-date_added')[:50])
for project in projects:
enriched_project = serialize(project)
enriched_project['organization'] = serialize(org)
keys = list(ProjectKey.objects.filter(
project=project,
roles=ProjectKey.roles.store,
status=ProjectKeyStatus.ACTIVE,
))
enriched_project['keys'] = serialize(keys)
filled_projects.append(enriched_project)
# Fetching or creating a token
token = None
tokens = [
x for x in ApiToken.objects.filter(user=request.user).all()
if 'project:releases' in x.get_scopes()
]
if not tokens:
token = ApiToken.objects.create(
user=request.user,
scope_list=['project:releases'],
refresh_token=None,
expires_at=None,
)
else:
token = tokens[0]
result = {
'apiKeys': serialize(token),
'projects': filled_projects
}
key = '%s%s' % (SETUP_WIZARD_CACHE_KEY, wizard_hash)
default_cache.set(key, result, SETUP_WIZARD_CACHE_TIMEOUT)
return render_to_response('sentry/setup-wizard.html', context, request)
def _do_process_event(cache_key, start_time, event_id, process_task, data=None):
from sentry.plugins import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "process"}, skip_internal=False
)
error_logger.error("process.failed.empty", extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(enhancer, data, _passthrough_errors=(RetrySymbolication,))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 3600:
raise RuntimeError("Event spent one hour in processing")
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data["project"] == project_id, "Project cannot be mutated by preprocessor"
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(
remove_other=False, is_renormalize=True, **DEFAULT_STORE_NORMALIZER_ARGS
)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id, start_time, data)
process_task.delay(cache_key, start_time=start_time, event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
try:
manager = EventManager(data)
event = manager.save(project_id)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments',
event.project.organization,
actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
except HashDiscarded:
increment_list = [
(tsdb.models.project_total_received_discarded, project_id),
]
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
increment_list.extend([
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_blacklisted,
project.organization_id),
])
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(
id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
else:
increment_list.append(
(tsdb.models.key_total_blacklisted, project_key.id))
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
tsdb.incr_multi(
increment_list,
timestamp=to_datetime(start_time)
if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
attachment_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def run_symbolicator(project,
request_id_cache_key,
create_task=create_payload_task,
**kwargs):
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
# Will be set lazily when a symbolicator request is fired
sources = None
attempts = 0
wait = 0.5
with sess:
while True:
try:
if request_id:
rv = _poll_symbolication_task(
sess=sess,
base_url=base_url,
request_id=request_id,
project_id=project_id,
)
else:
if sources is None:
sources = get_sources_for_project(project)
rv = create_task(sess=sess,
base_url=base_url,
project_id=project_id,
sources=sources,
**kwargs)
metrics.incr('events.symbolicator.status_code',
tags={
'status_code': rv.status_code,
'project_id': project_id,
})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response',
tags={
'response': json['status'],
'project_id': project_id,
})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
else:
default_cache.delete(request_id_cache_key)
return json
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
key_id = None if data is None else data.get('key_id')
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'post'},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments', event.project.organization, actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id
)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or \
features.has('organizations:event-attachments', event.project.organization, actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def _do_process_event(cache_key, start_time, event_id, process_task):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'process'},
skip_internal=False)
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project = data['project']
with configure_scope() as scope:
scope.set_tag("project", project)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project)
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in (enhancers or ()):
enhanced = safe_execute(enhancer, data)
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key, project, list(issues.values()),
event_id=event_id, start_time=start_time,
reprocessing_rev=reprocessing_rev
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
process_task.delay(cache_key, start_time=start_time,
event_id=event_id)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
default_cache.set(cache_key, data, 3600)
save_event.delay(
cache_key=cache_key, data=None, start_time=start_time, event_id=event_id,
project_id=project
)
def _do_process_event(cache_key, start_time, event_id, process_task,
data=None):
from sentry.plugins import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'process'},
skip_internal=False)
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
data = CanonicalKeyDict(data)
project_id = data['project']
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in (enhancers or ()):
enhanced = safe_execute(enhancer, data, _passthrough_errors=(RetrySymbolication,))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (time() - start_time) > 3600:
raise RuntimeError('Event spent one hour in processing')
retry_process_event.apply_async(
args=(),
kwargs={
'process_task_name': process_task.__name__,
'task_kwargs': {
'cache_key': cache_key,
'event_id': event_id,
'start_time': start_time,
}
},
countdown=e.retry_after
)
return
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project_id, 'Project cannot be mutated by preprocessor'
project = Project.objects.get_from_cache(id=project_id)
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key, project_id, list(issues.values()),
event_id=event_id, start_time=start_time,
reprocessing_rev=reprocessing_rev
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id, start_time, data)
process_task.delay(cache_key, start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
Raven.tags_context({
'project': project_id,
})
try:
manager = EventManager(data)
manager.save(project_id)
except HashDiscarded:
increment_list = [
(tsdb.models.project_total_received_discarded, project_id),
]
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
increment_list.extend([
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_blacklisted, project.organization_id),
])
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
else:
increment_list.append((tsdb.models.key_total_blacklisted, project_key.id))
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
tsdb.incr_multi(
increment_list,
timestamp=to_datetime(start_time) if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def _do_process_event(
cache_key,
start_time,
event_id,
process_task,
data=None,
data_has_changed=None,
new_process_behavior=None,
):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
has_changed = bool(data_has_changed)
new_process_behavior = bool(new_process_behavior)
metrics.incr("tasks.store.process_event.new_process_behavior",
tags={"value": new_process_behavior})
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
if not new_process_behavior:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
with metrics.timer("tasks.store.process_event.enhancers",
tags={"plugin": plugin.slug}):
enhancers = safe_execute(plugin.get_event_enhancers,
data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(
enhancer,
data,
_passthrough_errors=(RetrySymbolication, ))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
with metrics.timer("tasks.store.process_event.stacktraces"):
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning("process.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"process.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# *Right now* the only sensitive data that is added in stacktrace
# processing are usernames in filepaths, so we run directly after
# stacktrace processors and `get_event_enhancers`.
#
# We do not yet want to deal with context data produced by plugins like
# sessionstack or fullstory (which are in `get_event_preprocessors`), as
# this data is very unlikely to be sensitive data. This is why scrubbing
# happens somewhere in the middle of the pipeline.
#
# On the other hand, Javascript event error translation is happening after
# this block because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`.
#
# We are fairly confident, however, that this should run *before*
# re-normalization as it is hard to find sensitive data in partially
# trimmed strings.
if (has_changed and options.get("processing.can-use-scrubbers")
and features.has("organizations:datascrubbers-v2",
project.organization,
actor=None)):
with metrics.timer("tasks.store.datascrubbers.scrub"):
project_config = get_project_config(project)
new_data = safe_execute(scrub_data,
project_config=project_config,
event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
with metrics.timer("tasks.store.process_event.preprocessors",
tags={"plugin": plugin.slug}):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
_do_preprocess_event(cache_key, data, start_time, event_id,
process_task, project)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_process_event(cache_key, start_time, event_id, process_task):
from sentry.plugins import plugins
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'process'})
error_logger.error('process.failed.empty', extra={'cache_key': cache_key})
return
project = data['project']
Raven.tags_context({
'project': project,
})
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project)
# Stacktrace based event processors. These run before anything else.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(
plugin.get_event_preprocessors, data=data, _with_transaction=False
)
for processor in (processors or ()):
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data['project'] == project, 'Project cannot be mutated by preprocessor'
if has_changed:
issues = data.get('processing_issues')
try:
if issues and create_failed_event(
cache_key, project, list(issues.values()),
event_id=event_id, start_time=start_time,
reprocessing_rev=reprocessing_rev
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
process_task.delay(cache_key, start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
save_event.delay(
cache_key=cache_key, data=None, start_time=start_time, event_id=event_id,
project_id=project
)
def _do_process_event(cache_key,
start_time,
event_id,
process_task,
data=None):
from sentry.plugins.base import plugins
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "process"
},
skip_internal=False)
error_logger.error("process.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
with configure_scope() as scope:
scope.set_tag("project", project_id)
has_changed = False
# Fetch the reprocessing revision
reprocessing_rev = reprocessing.get_reprocessing_revision(project_id)
try:
# Event enhancers. These run before anything else.
for plugin in plugins.all(version=2):
enhancers = safe_execute(plugin.get_event_enhancers, data=data)
for enhancer in enhancers or ():
enhanced = safe_execute(
enhancer, data, _passthrough_errors=(RetrySymbolication, ))
if enhanced:
data = enhanced
has_changed = True
# Stacktrace based event processors.
new_data = process_stacktraces(data)
if new_data is not None:
has_changed = True
data = new_data
except RetrySymbolication as e:
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning("process.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"process.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
retry_process_event.apply_async(
args=(),
kwargs={
"process_task_name": process_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# Second round of datascrubbing after stacktrace and language-specific
# processing. First round happened as part of ingest.
#
# We assume that all potential PII is produced as part of stacktrace
# processors and event enhancers.
#
# We assume that plugins for eg sessionstack (running via
# `plugin.get_event_preprocessors`) are not producing data that should be
# PII-stripped, ever.
#
# XXX(markus): Javascript event error translation is happening after this block
# because it uses `get_event_preprocessors` instead of
# `get_event_enhancers`, possibly move?
if has_changed and features.has("organizations:datascrubbers-v2",
project.organization,
actor=None):
with metrics.timer("tasks.store.datascrubbers.scrub"):
project_config = get_project_config(project)
new_data = safe_execute(scrub_data,
project_config=project_config,
event=data.data)
# XXX(markus): When datascrubbing is finally "totally stable", we might want
# to drop the event if it crashes to avoid saving PII
if new_data is not None:
data.data = new_data
# TODO(dcramer): ideally we would know if data changed by default
# Default event processors.
for plugin in plugins.all(version=2):
processors = safe_execute(plugin.get_event_preprocessors,
data=data,
_with_transaction=False)
for processor in processors or ():
result = safe_execute(processor, data)
if result:
data = result
has_changed = True
assert data[
"project"] == project_id, "Project cannot be mutated by plugins"
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
# Run some of normalization again such that we don't:
# - persist e.g. incredibly large stacktraces from minidumps
# - store event timestamps that are older than our retention window
# (also happening with minidumps)
normalizer = StoreNormalizer(remove_other=False,
is_renormalize=True,
**DEFAULT_STORE_NORMALIZER_ARGS)
data = normalizer.normalize_event(dict(data))
issues = data.get("processing_issues")
try:
if issues and create_failed_event(
cache_key,
data,
project_id,
list(issues.values()),
event_id=event_id,
start_time=start_time,
reprocessing_rev=reprocessing_rev,
):
return
except RetryProcessing:
# If `create_failed_event` indicates that we need to retry we
# invoke outselves again. This happens when the reprocessing
# revision changed while we were processing.
from_reprocessing = process_task is process_event_from_reprocessing
submit_process(project, from_reprocessing, cache_key, event_id,
start_time, data)
process_task.delay(cache_key,
start_time=start_time,
event_id=event_id)
return
default_cache.set(cache_key, data, 3600)
submit_save_event(project, cache_key, event_id, start_time, data)
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer(
"tasks.store.do_save_event.get_cache") as metric_tags:
data = default_cache.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get(
"type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
event = manager.save(project_id,
assume_normalized=True,
start_time=start_time,
cache_key=cache_key)
except HashDiscarded:
pass
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
default_cache.delete(cache_key)
with metrics.timer(
"tasks.store.do_save_event.delete_attachment_cache"):
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or features.has(
"organizations:event-attachments",
event.project.organization,
actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])
def post(self, request):
"""
Registers a Relay
`````````````````
Registers the relay with the sentry installation. If a relay boots
it will always attempt to invoke this endpoint.
"""
try:
json_data = json.loads(request.body)
except ValueError:
return Response({
'detail': 'No valid json body',
}, status=status.HTTP_400_BAD_REQUEST)
serializer = RelayRegisterResponseSerializer(data=json_data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
sig = get_header_relay_signature(request)
if not sig:
return Response({
'detail': 'Missing relay signature',
}, status=status.HTTP_400_BAD_REQUEST)
relay_id = six.text_type(get_register_response_relay_id(request.body))
if relay_id != get_header_relay_id(request):
return Response({
'detail': 'relay_id in payload did not match header',
}, status=status.HTTP_400_BAD_REQUEST)
params = default_cache.get('relay-auth:%s' % relay_id)
if params is None:
return Response({
'detail': 'Challenge expired'
}, status=status.HTTP_401_UNAUTHORIZED)
key = PublicKey.parse(params['public_key'])
try:
validate_register_response(key, request.body, sig)
except Exception as exc:
return Response({
'detail': str(exc).splitlines()[0],
}, status=status.HTTP_400_BAD_REQUEST)
is_internal = is_internal_relay(request, params['public_key'])
try:
relay = Relay.objects.get(relay_id=relay_id)
except Relay.DoesNotExist:
relay = Relay.objects.create(
relay_id=relay_id,
public_key=params['public_key'],
is_internal=is_internal
)
else:
relay.last_seen = timezone.now()
relay.is_internal = is_internal
relay.save()
default_cache.delete('relay-auth:%s' % relay_id)
return Response(serialize({
'relay_id': relay.relay_id,
}))
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None,
reprocessing_rev=None):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
reprocessing_active = ProjectOption.objects.get_value(
project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT
)
# In case there is reprocessing active but the current reprocessing
# revision is already different than when we started, we want to
# immediately retry the event. This resolves the problem when
# otherwise a concurrent change of debug symbols might leave a
# reprocessing issue stuck in the project forever.
if reprocessing_active and \
reprocessing.get_reprocessing_revision(project_id, cached=False) != \
reprocessing_rev:
raise RetryProcessing()
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, 'sentry:sent_failed_event_hint', False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={'reprocessing_active': reprocessing_active,
'issues': issues},
).send_notification()
ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'}, skip_internal=False)
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return True
data = CanonicalKeyDict(data)
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
return True
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
key_id = None if data is None else data.get('key_id')
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'post'},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments', event.project.organization, actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id
)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or \
features.has('organizations:event-attachments', event.project.organization, actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def _do_symbolicate_event(cache_key,
start_time,
event_id,
symbolicate_task,
data=None):
from sentry.lang.native.processing import get_symbolication_function
if data is None:
data = default_cache.get(cache_key)
if data is None:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "symbolicate"
},
skip_internal=False)
error_logger.error("symbolicate.failed.empty",
extra={"cache_key": cache_key})
return
data = CanonicalKeyDict(data)
project_id = data["project"]
set_current_project(project_id)
event_id = data["event_id"]
project = Project.objects.get_from_cache(id=project_id)
symbolication_function = get_symbolication_function(data)
has_changed = False
from_reprocessing = symbolicate_task is symbolicate_event_from_reprocessing
try:
with metrics.timer("tasks.store.symbolicate_event.symbolication"):
symbolicated_data = safe_execute(
symbolication_function,
data,
_passthrough_errors=(RetrySymbolication, ))
if symbolicated_data:
data = symbolicated_data
has_changed = True
except RetrySymbolication as e:
error_logger.warn("retry symbolication")
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_WARN_TIMEOUT:
error_logger.warning("symbolicate.slow",
extra={
"project_id": project_id,
"event_id": event_id
})
if start_time and (
time() -
start_time) > settings.SYMBOLICATOR_PROCESS_EVENT_HARD_TIMEOUT:
# Do not drop event but actually continue with rest of pipeline
# (persisting unsymbolicated event)
error_logger.exception(
"symbolicate.failed.infinite_retry",
extra={
"project_id": project_id,
"event_id": event_id
},
)
else:
# Requeue the task in the "sleep" queue
retry_symbolicate_event.apply_async(
args=(),
kwargs={
"symbolicate_task_name": symbolicate_task.__name__,
"task_kwargs": {
"cache_key": cache_key,
"event_id": event_id,
"start_time": start_time,
},
},
countdown=e.retry_after,
)
return
# We cannot persist canonical types in the cache, so we need to
# downgrade this.
if isinstance(data, CANONICAL_TYPES):
data = dict(data.items())
if has_changed:
default_cache.set(cache_key, data, 3600)
submit_process(
project,
from_reprocessing,
cache_key,
event_id,
start_time,
data,
has_changed,
new_process_behavior=True,
)
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
from sentry.ingest.outcomes_consumer import mark_signal_sent
event_type = "none"
if cache_key and data is None:
with metrics.timer(
"tasks.store.do_save_event.get_cache") as metric_tags:
data = default_cache.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get(
"type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
key_id = None if data is None else data.get("key_id")
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
event = manager.save(project_id,
assume_normalized=True,
cache_key=cache_key)
with metrics.timer("tasks.store.do_save_event.track_outcome"):
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id,
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
# There is no signal supposed to be sent for this particular
# outcome-reason combination. Prevent the outcome consumer from
# emitting it for now.
#
# XXX(markus): Revisit decision about signals once outcomes consumer is stable.
mark_signal_sent(project_id, event_id)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id,
)
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
default_cache.delete(cache_key)
with metrics.timer(
"tasks.store.do_save_event.delete_attachment_cache"):
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or features.has(
"organizations:event-attachments",
event.project.organization,
actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])
def run_symbolicator(project, request_id_cache_key, create_task=create_payload_task, **kwargs):
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
# Will be set lazily when a symbolicator request is fired
sources = None
attempts = 0
wait = 0.5
with sess:
while True:
try:
if request_id:
rv = _poll_symbolication_task(
sess=sess, base_url=base_url,
request_id=request_id, project_id=project_id,
)
else:
if sources is None:
sources = get_sources_for_project(project)
rv = create_task(
sess=sess, base_url=base_url,
project_id=project_id,
sources=sources,
**kwargs
)
metrics.incr('events.symbolicator.status_code', tags={
'status_code': rv.status_code,
'project_id': project_id,
})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response', tags={
'response': json['status'],
'project_id': project_id,
})
if json['status'] == 'pending':
default_cache.set(
request_id_cache_key,
json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator', exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0