def save_event(cache_key=None, data=None, start_time=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
project = data.pop('project')
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def post(self, request):
"""
Registers a Relay
`````````````````
Registers the relay with the sentry installation. If a relay boots
it will always attempt to invoke this endpoint.
"""
try:
json_data = json.loads(request.body)
except ValueError:
return Response({"detail": "No valid json body"},
status=status.HTTP_400_BAD_REQUEST)
serializer = RelayRegisterResponseSerializer(data=json_data)
if not serializer.is_valid():
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
sig = get_header_relay_signature(request)
if not sig:
return Response({"detail": "Missing relay signature"},
status=status.HTTP_400_BAD_REQUEST)
relay_id = six.text_type(get_register_response_relay_id(request.body))
if relay_id != get_header_relay_id(request):
return Response(
{"detail": "relay_id in payload did not match header"},
status=status.HTTP_400_BAD_REQUEST,
)
params = default_cache.get("relay-auth:%s" % relay_id)
if params is None:
return Response({"detail": "Challenge expired"},
status=status.HTTP_401_UNAUTHORIZED)
key = PublicKey.parse(params["public_key"])
try:
validate_register_response(key, request.body, sig)
except Exception as exc:
return Response({"detail": str(exc).splitlines()[0]},
status=status.HTTP_400_BAD_REQUEST)
is_internal = is_internal_relay(request, params["public_key"])
try:
relay = Relay.objects.get(relay_id=relay_id)
except Relay.DoesNotExist:
relay = Relay.objects.create(relay_id=relay_id,
public_key=params["public_key"],
is_internal=is_internal)
else:
relay.last_seen = timezone.now()
relay.is_internal = is_internal
relay.save()
default_cache.delete("relay-auth:%s" % relay_id)
return Response(serialize({"relay_id": relay.relay_id}))
def create_failed_event(cache_key, project, issues, event_id):
"""If processing failed we put the original data from the cache into a
raw event.
"""
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'})
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project,
event_id=event_id,
datetime=datetime.utcfromtimestamp(
data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
project = data.pop('project')
delete_raw_event(project, event_id)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process', time() - start_time,
instance=data['platform'])
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
reprocessing_active = ProjectOption.objects.get_value(
project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT
)
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, 'sentry:sent_failed_event_hint', False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={'reprocessing_active': reprocessing_active,
'issues': issues},
).send_notification()
ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'})
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return True
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
return True
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
reprocessing_active = ProjectOption.objects.get_value(
project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT
)
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, 'sentry:sent_failed_event_hint', False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={'reprocessing_active': reprocessing_active,
'issues': issues},
).send_notification()
ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'})
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return True
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
return True
def delete(self, request, wizard_hash=None):
"""
This removes the cache content for a specific hash
"""
if wizard_hash is not None:
key = "%s%s" % (SETUP_WIZARD_CACHE_KEY, wizard_hash)
default_cache.delete(key)
return Response(status=200)
def delete(self, request: Request, wizard_hash=None) -> Response:
"""
This removes the cache content for a specific hash
"""
if wizard_hash is not None:
key = f"{SETUP_WIZARD_CACHE_KEY}{wizard_hash}"
default_cache.delete(key)
return Response(status=200)
def delete(self, request, wizard_hash=None):
"""
This removes the cache content for a specific hash
"""
if wizard_hash is not None:
key = '%s%s' % (SETUP_WIZARD_CACHE_KEY, wizard_hash)
default_cache.delete(key)
return Response(status=200)
def set_assemble_status(project, checksum, state, detail=None):
cache_key = 'assemble-status:%s' % _get_idempotency_id(project, checksum)
# If the state is okay we actually clear it from the cache because in
# that case a project dsym file was created.
if state == ChunkFileState.OK:
default_cache.delete(cache_key)
else:
default_cache.set(cache_key, (state, detail), 300)
def save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import tsdb
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
project = data.pop('project')
delete_raw_event(project, event_id, allow_hint_clear=True)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
except HashDiscarded as exc:
# TODO(jess): remove this before it goes out to a wider audience
info_logger.info('discarded.hash',
extra={
'project_id': project,
'description': exc.message,
})
tsdb.incr(
tsdb.models.project_total_received_discarded,
project,
timestamp=to_datetime(start_time)
if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def set_assemble_status(project, checksum, state, detail=None):
cache_key = 'assemble-status:%s' % _get_idempotency_id(
project, checksum)
# If the state is okay we actually clear it from the cache because in
# that case a project dsym file was created.
if state == ChunkFileState.OK:
default_cache.delete(cache_key)
else:
default_cache.set(cache_key, (state, detail), 300)
def _process(self, create_task, task_name):
task_id = default_cache.get(self.task_id_cache_key)
json_response = None
with self.sess:
try:
if task_id:
# Processing has already started and we need to poll
# symbolicator for an update. This in turn may put us back into
# the queue.
json_response = self.sess.query_task(task_id)
if json_response is None:
# This is a new task, so we compute all request parameters
# (potentially expensive if we need to pull minidumps), and then
# upload all information to symbolicator. It will likely not
# have a response ready immediately, so we start polling after
# some timeout.
json_response = create_task()
except ServiceUnavailable:
# 503 can indicate that symbolicator is restarting. Wait for a
# reboot, then try again. This overrides the default behavior of
# retrying after just a second.
#
# If there is no response attached, it's a connection error.
raise RetrySymbolication(
retry_after=settings.SYMBOLICATOR_MAX_RETRY_AFTER)
metrics.incr(
"events.symbolicator.response",
tags={
"response": json_response.get("status") or "null",
"task_name": task_name
},
)
# Symbolication is still in progress. Bail out and try again
# after some timeout. Symbolicator keeps the response for the
# first one to poll it.
if json_response["status"] == "pending":
default_cache.set(self.task_id_cache_key,
json_response["request_id"],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(
retry_after=json_response["retry_after"])
else:
# Once we arrive here, we are done processing. Clean up the
# task id from the cache.
default_cache.delete(self.task_id_cache_key)
metrics.timing("events.symbolicator.response.completed.size",
len(json.dumps(json_response)))
reverse_source_aliases(json_response)
redact_internal_sources(json_response)
return json_response
def save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
project = data.pop('project')
delete_raw_event(project, event_id, allow_hint_clear=True)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
except HashDiscarded as exc:
info_logger.info('discarded.hash',
extra={
'project_id': project.id,
'message': exc.message,
})
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def _process(self, create_task):
task_id = default_cache.get(self.task_id_cache_key)
json = None
with self.sess:
try:
if task_id:
# Processing has already started and we need to poll
# symbolicator for an update. This in turn may put us back into
# the queue.
json = self.sess.query_task(task_id)
if json is None:
# This is a new task, so we compute all request parameters
# (potentially expensive if we need to pull minidumps), and then
# upload all information to symbolicator. It will likely not
# have a response ready immediately, so we start polling after
# some timeout.
json = create_task()
except ServiceUnavailable:
# 503 can indicate that symbolicator is restarting. Wait for a
# reboot, then try again. This overrides the default behavior of
# retrying after just a second.
#
# If there is no response attached, it's a connection error.
raise RetrySymbolication(retry_after=10)
metrics.incr('events.symbolicator.response', tags={
'response': json.get('status') or 'null',
'project_id': self.sess.project_id,
})
# Symbolication is still in progress. Bail out and try again
# after some timeout. Symbolicator keeps the response for the
# first one to poll it.
if json['status'] == 'pending':
default_cache.set(
self.task_id_cache_key,
json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
else:
# Once we arrive here, we are done processing. Clean up the
# task id from the cache.
default_cache.delete(self.task_id_cache_key)
return json
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import tsdb
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
project = data.pop('project')
delete_raw_event(project, event_id, allow_hint_clear=True)
Raven.tags_context({
'project': project,
})
try:
manager = EventManager(data)
manager.save(project)
except HashDiscarded as exc:
# TODO(jess): remove this before it goes out to a wider audience
info_logger.info(
'discarded.hash', extra={
'project_id': project,
'description': exc.message,
}
)
tsdb.incr(tsdb.models.project_total_received_discarded, project, timestamp=start_time)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def save_event(cache_key=None, data=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if data is None:
return
project = data.pop('project')
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
def save_event(cache_key=None, data=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if data is None:
return
project = data.pop('project')
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
def save_event(cache_key=None, data=None, start_time=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import EventManager
if cache_key:
data = default_cache.get(cache_key)
if data is None:
return
project = data.pop('project')
try:
manager = EventManager(data)
manager.save(project)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process', time() - start_time)
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager, track_outcome
from sentry import quotas
from sentry.models import ProjectKey
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
key_id = None if data is None else data.get('key_id')
timestamp = to_datetime(start_time) if start_time is not None else None
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments',
event.project.organization,
actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(event.project.organization_id, event.project.id, key_id,
'accepted', None, timestamp)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
track_outcome(project.organization_id, project_id, key_id, 'filtered',
reason, timestamp)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or \
features.has('organizations:event-attachments', event.project.organization, actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def create_failed_event(cache_key,
project_id,
issues,
event_id,
start_time=None,
reprocessing_rev=None):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
reprocessing_active = ProjectOption.objects.get_value(
project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT)
# In case there is reprocessing active but the current reprocessing
# revision is already different than when we started, we want to
# immediately retry the event. This resolves the problem when
# otherwise a concurrent change of debug symbols might leave a
# reprocessing issue stuck in the project forever.
if reprocessing_active and \
reprocessing.get_reprocessing_revision(project_id, cached=False) != \
reprocessing_rev:
raise RetryProcessing()
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, 'sentry:sent_failed_event_hint', False)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={
'reprocessing_active': reprocessing_active,
'issues': issues
},
).send_notification()
ProjectOption.objects.set_value(project,
'sentry:sent_failed_event_hint', True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'raw'
},
skip_internal=False)
error_logger.error('process.failed_raw.empty',
extra={'cache_key': cache_key})
return True
data = CanonicalKeyDict(data)
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(
data['timestamp']).replace(tzinfo=timezone.utc),
data=data)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
return True
def run_symbolicator(stacktraces, modules, project, arch, signal,
request_id_cache_key):
internal_url_prefix = options.get('system.internal-url-prefix') \
or options.get('system.url-prefix')
assert internal_url_prefix
sentry_source_url = '%s%s' % (
internal_url_prefix.rstrip('/'),
reverse('sentry-api-0-dsym-files',
kwargs={
'organization_slug': project.organization.slug,
'project_slug': project.slug
}))
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
attempts = 0
wait = 0.5
with sess:
while 1:
try:
if request_id:
rv = _poll_symbolication_task(sess=sess,
base_url=base_url,
request_id=request_id)
else:
rv = _create_symbolication_task(
sess=sess,
base_url=base_url,
project_id=project_id,
sentry_source_url=sentry_source_url,
signal=signal,
stacktraces=stacktraces,
modules=modules)
metrics.incr('events.symbolicator.status.%s' % rv.status_code,
tags={'project_id': project_id})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response.%s' %
json['status'],
tags={'project_id': project_id})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
def save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
Raven.tags_context({
'project': project_id,
})
try:
manager = EventManager(data)
event = manager.save(project_id)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments',
event.project.organization,
actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
except HashDiscarded:
increment_list = [
(tsdb.models.project_total_received_discarded, project_id),
]
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
increment_list.extend([
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_blacklisted,
project.organization_id),
])
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(
id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
else:
increment_list.append(
(tsdb.models.key_total_blacklisted, project_key.id))
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
tsdb.incr_multi(
increment_list,
timestamp=to_datetime(start_time)
if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
attachment_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def post(self, request):
"""
Registers a Relay
`````````````````
Registers the relay with the sentry installation. If a relay boots
it will always attempt to invoke this endpoint.
"""
try:
json_data = json.loads(request.body)
except ValueError:
return Response({
'detail': 'No valid json body',
}, status=status.HTTP_400_BAD_REQUEST)
serializer = RelayRegisterResponseSerializer(data=json_data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
sig = get_header_relay_signature(request)
if not sig:
return Response({
'detail': 'Missing relay signature',
}, status=status.HTTP_400_BAD_REQUEST)
relay_id = six.text_type(get_register_response_relay_id(request.body))
if relay_id != get_header_relay_id(request):
return Response({
'detail': 'relay_id in payload did not match header',
}, status=status.HTTP_400_BAD_REQUEST)
params = default_cache.get('relay-auth:%s' % relay_id)
if params is None:
return Response({
'detail': 'Challenge expired'
}, status=status.HTTP_401_UNAUTHORIZED)
key = PublicKey.parse(params['public_key'])
try:
validate_register_response(key, request.body, sig)
except Exception as exc:
return Response({
'detail': str(exc).splitlines()[0],
}, status=status.HTTP_400_BAD_REQUEST)
is_internal = is_internal_relay(request, params['public_key'])
try:
relay = Relay.objects.get(relay_id=relay_id)
except Relay.DoesNotExist:
relay = Relay.objects.create(
relay_id=relay_id,
public_key=params['public_key'],
is_internal=is_internal
)
else:
relay.last_seen = timezone.now()
relay.is_internal = is_internal
relay.save()
default_cache.delete('relay-auth:%s' % relay_id)
return Response(serialize({
'relay_id': relay.relay_id,
}))
def run_symbolicator(stacktraces, modules, project, arch, signal,
request_id_cache_key):
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
# Will be set lazily when a symbolicator request is fired
sources = None
attempts = 0
wait = 0.5
with sess:
while True:
try:
if request_id:
rv = _poll_symbolication_task(sess=sess,
base_url=base_url,
request_id=request_id)
else:
if sources is None:
sources = get_sources_for_project(project)
rv = _create_symbolication_task(sess=sess,
base_url=base_url,
project_id=project_id,
sources=sources,
signal=signal,
stacktraces=stacktraces,
modules=modules)
metrics.incr('events.symbolicator.status_code',
tags={
'status_code': rv.status_code,
'project_id': project_id,
})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response',
tags={
'response': json['status'],
'project_id': project_id,
})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
def save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'})
return
Raven.tags_context({
'project': project_id,
})
try:
manager = EventManager(data)
manager.save(project_id)
except HashDiscarded:
increment_list = [
(tsdb.models.project_total_received_discarded, project_id),
]
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
increment_list.extend([
(tsdb.models.project_total_blacklisted, project.id),
(tsdb.models.organization_total_blacklisted, project.organization_id),
])
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
else:
increment_list.append((tsdb.models.key_total_blacklisted, project_key.id))
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
tsdb.incr_multi(
increment_list,
timestamp=to_datetime(start_time) if start_time is not None else None,
)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def _do_save_event(
cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs
):
"""
Saves an event to the database.
"""
set_current_project(project_id)
from sentry.event_manager import EventManager, HashDiscarded
event_type = "none"
if cache_key and data is None:
with metrics.timer("tasks.store.do_save_event.get_cache") as metric_tags:
data = default_cache.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get("type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
set_current_project(project_id)
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
"events.failed", tags={"reason": "cache", "stage": "post"}, skip_internal=False
)
return
event = None
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
event = manager.save(
project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key
)
except HashDiscarded:
pass
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
default_cache.delete(cache_key)
with metrics.timer("tasks.store.do_save_event.delete_attachment_cache"):
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or features.has(
"organizations:event-attachments", event.project.organization, actor=None
):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
"events.time-to-process", time() - start_time, instance=data["platform"]
)
def run_symbolicator(project, request_id_cache_key, create_task=create_payload_task, **kwargs):
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
# Will be set lazily when a symbolicator request is fired
sources = None
attempts = 0
wait = 0.5
with sess:
while True:
try:
if request_id:
rv = _poll_symbolication_task(
sess=sess, base_url=base_url,
request_id=request_id, project_id=project_id,
)
else:
if sources is None:
sources = get_sources_for_project(project)
rv = create_task(
sess=sess, base_url=base_url,
project_id=project_id,
sources=sources,
**kwargs
)
metrics.incr('events.symbolicator.status_code', tags={
'status_code': rv.status_code,
'project_id': project_id,
})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response', tags={
'response': json['status'],
'project_id': project_id,
})
if json['status'] == 'pending':
default_cache.set(
request_id_cache_key,
json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator', exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None,
project_id=None, **kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
if cache_key and data is None:
data = default_cache.get(cache_key)
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data['event_id']
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop('project')
key_id = None if data is None else data.get('key_id')
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr(
'events.failed',
tags={
'reason': 'cache',
'stage': 'post'},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
manager = EventManager(data)
event = manager.save(project_id, assume_normalized=True)
# Always load attachments from the cache so we can later prune them.
# Only save them if the event-attachments feature is active, though.
if features.has('organizations:event-attachments', event.project.organization, actor=None):
attachments = attachment_cache.get(cache_key) or []
for attachment in attachments:
save_attachment(event, attachment)
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id
)
finally:
if cache_key:
default_cache.delete(cache_key)
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or \
features.has('organizations:event-attachments', event.project.organization, actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing(
'events.time-to-process',
time() - start_time,
instance=data['platform'])
def save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas, tsdb
from sentry.models import ProjectKey
if cache_key:
data = default_cache.get(cache_key)
if event_id is None and data is not None:
event_id = data['event_id']
if data is None:
metrics.incr('events.failed',
tags={
'reason': 'cache',
'stage': 'post'
})
return
project_id = data.pop('project')
delete_raw_event(project_id, event_id, allow_hint_clear=True)
Raven.tags_context({
'project': project_id,
})
try:
manager = EventManager(data)
manager.save(project_id)
except HashDiscarded:
tsdb.incr(
tsdb.models.project_total_received_discarded,
project_id,
timestamp=to_datetime(start_time)
if start_time is not None else None,
)
try:
project = Project.objects.get_from_cache(id=project_id)
except Project.DoesNotExist:
pass
else:
project_key = None
if data.get('key_id') is not None:
try:
project_key = ProjectKey.objects.get_from_cache(
id=data['key_id'])
except ProjectKey.DoesNotExist:
pass
quotas.refund(
project,
key=project_key,
timestamp=start_time,
)
finally:
if cache_key:
default_cache.delete(cache_key)
if start_time:
metrics.timing('events.time-to-process',
time() - start_time,
instance=data['platform'])
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None,
reprocessing_rev=None):
"""If processing failed we put the original data from the cache into a
raw event. Returns `True` if a failed event was inserted
"""
reprocessing_active = ProjectOption.objects.get_value(
project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT
)
# In case there is reprocessing active but the current reprocessing
# revision is already different than when we started, we want to
# immediately retry the event. This resolves the problem when
# otherwise a concurrent change of debug symbols might leave a
# reprocessing issue stuck in the project forever.
if reprocessing_active and \
reprocessing.get_reprocessing_revision(project_id, cached=False) != \
reprocessing_rev:
raise RetryProcessing()
# The first time we encounter a failed event and the hint was cleared
# we send a notification.
sent_notification = ProjectOption.objects.get_value(
project_id, 'sentry:sent_failed_event_hint', False
)
if not sent_notification:
project = Project.objects.get_from_cache(id=project_id)
Activity.objects.create(
type=Activity.NEW_PROCESSING_ISSUES,
project=project,
datetime=to_datetime(start_time),
data={'reprocessing_active': reprocessing_active,
'issues': issues},
).send_notification()
ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True)
# If reprocessing is not active we bail now without creating the
# processing issues
if not reprocessing_active:
return False
# We need to get the original data here instead of passing the data in
# from the last processing step because we do not want any
# modifications to take place.
delete_raw_event(project_id, event_id)
data = default_cache.get(cache_key)
if data is None:
metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'}, skip_internal=False)
error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key})
return True
data = CanonicalKeyDict(data)
from sentry.models import RawEvent, ProcessingIssue
raw_event = RawEvent.objects.create(
project_id=project_id,
event_id=event_id,
datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc),
data=data
)
for issue in issues:
ProcessingIssue.objects.record_processing_issue(
raw_event=raw_event,
scope=issue['scope'],
object=issue['object'],
type=issue['type'],
data=issue['data'],
)
default_cache.delete(cache_key)
return True
def _do_save_event(cache_key=None,
data=None,
start_time=None,
event_id=None,
project_id=None,
**kwargs):
"""
Saves an event to the database.
"""
from sentry.event_manager import HashDiscarded, EventManager
from sentry import quotas
from sentry.models import ProjectKey
from sentry.utils.outcomes import Outcome, track_outcome
from sentry.ingest.outcomes_consumer import mark_signal_sent
event_type = "none"
if cache_key and data is None:
with metrics.timer(
"tasks.store.do_save_event.get_cache") as metric_tags:
data = default_cache.get(cache_key)
if data is not None:
metric_tags["event_type"] = event_type = data.get(
"type") or "none"
with metrics.global_tags(event_type=event_type):
if data is not None:
data = CanonicalKeyDict(data)
if event_id is None and data is not None:
event_id = data["event_id"]
# only when we come from reprocessing we get a project_id sent into
# the task.
if project_id is None:
project_id = data.pop("project")
key_id = None if data is None else data.get("key_id")
if key_id is not None:
key_id = int(key_id)
timestamp = to_datetime(start_time) if start_time is not None else None
# We only need to delete raw events for events that support
# reprocessing. If the data cannot be found we want to assume
# that we need to delete the raw event.
if not data or reprocessing.event_supports_reprocessing(data):
with metrics.timer("tasks.store.do_save_event.delete_raw_event"):
delete_raw_event(project_id, event_id, allow_hint_clear=True)
# This covers two cases: where data is None because we did not manage
# to fetch it from the default cache or the empty dictionary was
# stored in the default cache. The former happens if the event
# expired while being on the queue, the second happens on reprocessing
# if the raw event was deleted concurrently while we held on to
# it. This causes the node store to delete the data and we end up
# fetching an empty dict. We could in theory not invoke `save_event`
# in those cases but it's important that we always clean up the
# reprocessing reports correctly or they will screw up the UI. So
# to future proof this correctly we just handle this case here.
if not data:
metrics.incr("events.failed",
tags={
"reason": "cache",
"stage": "post"
},
skip_internal=False)
return
with configure_scope() as scope:
scope.set_tag("project", project_id)
event = None
try:
with metrics.timer("tasks.store.do_save_event.event_manager.save"):
manager = EventManager(data)
# event.project.organization is populated after this statement.
event = manager.save(project_id,
assume_normalized=True,
cache_key=cache_key)
with metrics.timer("tasks.store.do_save_event.track_outcome"):
# This is where we can finally say that we have accepted the event.
track_outcome(
event.project.organization_id,
event.project.id,
key_id,
Outcome.ACCEPTED,
None,
timestamp,
event_id,
)
except HashDiscarded:
project = Project.objects.get_from_cache(id=project_id)
reason = FilterStatKeys.DISCARDED_HASH
project_key = None
try:
if key_id is not None:
project_key = ProjectKey.objects.get_from_cache(id=key_id)
except ProjectKey.DoesNotExist:
pass
quotas.refund(project, key=project_key, timestamp=start_time)
# There is no signal supposed to be sent for this particular
# outcome-reason combination. Prevent the outcome consumer from
# emitting it for now.
#
# XXX(markus): Revisit decision about signals once outcomes consumer is stable.
mark_signal_sent(project_id, event_id)
track_outcome(
project.organization_id,
project_id,
key_id,
Outcome.FILTERED,
reason,
timestamp,
event_id,
)
finally:
if cache_key:
with metrics.timer("tasks.store.do_save_event.delete_cache"):
default_cache.delete(cache_key)
with metrics.timer(
"tasks.store.do_save_event.delete_attachment_cache"):
# For the unlikely case that we did not manage to persist the
# event we also delete the key always.
if event is None or features.has(
"organizations:event-attachments",
event.project.organization,
actor=None):
attachment_cache.delete(cache_key)
if start_time:
metrics.timing("events.time-to-process",
time() - start_time,
instance=data["platform"])