def save_event(cache_key=None, data=None, start_time=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import EventManager if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project = data.pop('project') Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def post(self, request): """ Registers a Relay ````````````````` Registers the relay with the sentry installation. If a relay boots it will always attempt to invoke this endpoint. """ try: json_data = json.loads(request.body) except ValueError: return Response({"detail": "No valid json body"}, status=status.HTTP_400_BAD_REQUEST) serializer = RelayRegisterResponseSerializer(data=json_data) if not serializer.is_valid(): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) sig = get_header_relay_signature(request) if not sig: return Response({"detail": "Missing relay signature"}, status=status.HTTP_400_BAD_REQUEST) relay_id = six.text_type(get_register_response_relay_id(request.body)) if relay_id != get_header_relay_id(request): return Response( {"detail": "relay_id in payload did not match header"}, status=status.HTTP_400_BAD_REQUEST, ) params = default_cache.get("relay-auth:%s" % relay_id) if params is None: return Response({"detail": "Challenge expired"}, status=status.HTTP_401_UNAUTHORIZED) key = PublicKey.parse(params["public_key"]) try: validate_register_response(key, request.body, sig) except Exception as exc: return Response({"detail": str(exc).splitlines()[0]}, status=status.HTTP_400_BAD_REQUEST) is_internal = is_internal_relay(request, params["public_key"]) try: relay = Relay.objects.get(relay_id=relay_id) except Relay.DoesNotExist: relay = Relay.objects.create(relay_id=relay_id, public_key=params["public_key"], is_internal=is_internal) else: relay.last_seen = timezone.now() relay.is_internal = is_internal relay.save() default_cache.delete("relay-auth:%s" % relay_id) return Response(serialize({"relay_id": relay.relay_id}))
def create_failed_event(cache_key, project, issues, event_id): """If processing failed we put the original data from the cache into a raw event. """ # We need to get the original data here instead of passing the data in # from the last processing step because we do not want any # modifications to take place. delete_raw_event(project, event_id) data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'}) error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key}) return from sentry.models import RawEvent, ProcessingIssue raw_event = RawEvent.objects.create( project_id=project, event_id=event_id, datetime=datetime.utcfromtimestamp( data['timestamp']).replace(tzinfo=timezone.utc), data=data ) for issue in issues: ProcessingIssue.objects.record_processing_issue( raw_event=raw_event, scope=issue['scope'], object=issue['object'], type=issue['type'], data=issue['data'], ) default_cache.delete(cache_key)
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import EventManager if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'}) return project = data.pop('project') delete_raw_event(project, event_id) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None): """If processing failed we put the original data from the cache into a raw event. Returns `True` if a failed event was inserted """ reprocessing_active = ProjectOption.objects.get_value( project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT ) # The first time we encounter a failed event and the hint was cleared # we send a notification. sent_notification = ProjectOption.objects.get_value( project_id, 'sentry:sent_failed_event_hint', False ) if not sent_notification: project = Project.objects.get_from_cache(id=project_id) Activity.objects.create( type=Activity.NEW_PROCESSING_ISSUES, project=project, datetime=to_datetime(start_time), data={'reprocessing_active': reprocessing_active, 'issues': issues}, ).send_notification() ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True) # If reprocessing is not active we bail now without creating the # processing issues if not reprocessing_active: return False # We need to get the original data here instead of passing the data in # from the last processing step because we do not want any # modifications to take place. delete_raw_event(project_id, event_id) data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'}) error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key}) return True from sentry.models import RawEvent, ProcessingIssue raw_event = RawEvent.objects.create( project_id=project_id, event_id=event_id, datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc), data=data ) for issue in issues: ProcessingIssue.objects.record_processing_issue( raw_event=raw_event, scope=issue['scope'], object=issue['object'], type=issue['type'], data=issue['data'], ) default_cache.delete(cache_key) return True
def delete(self, request, wizard_hash=None): """ This removes the cache content for a specific hash """ if wizard_hash is not None: key = "%s%s" % (SETUP_WIZARD_CACHE_KEY, wizard_hash) default_cache.delete(key) return Response(status=200)
def delete(self, request: Request, wizard_hash=None) -> Response: """ This removes the cache content for a specific hash """ if wizard_hash is not None: key = f"{SETUP_WIZARD_CACHE_KEY}{wizard_hash}" default_cache.delete(key) return Response(status=200)
def delete(self, request, wizard_hash=None): """ This removes the cache content for a specific hash """ if wizard_hash is not None: key = '%s%s' % (SETUP_WIZARD_CACHE_KEY, wizard_hash) default_cache.delete(key) return Response(status=200)
def set_assemble_status(project, checksum, state, detail=None): cache_key = 'assemble-status:%s' % _get_idempotency_id(project, checksum) # If the state is okay we actually clear it from the cache because in # that case a project dsym file was created. if state == ChunkFileState.OK: default_cache.delete(cache_key) else: default_cache.set(cache_key, (state, detail), 300)
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import tsdb if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project = data.pop('project') delete_raw_event(project, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) except HashDiscarded as exc: # TODO(jess): remove this before it goes out to a wider audience info_logger.info('discarded.hash', extra={ 'project_id': project, 'description': exc.message, }) tsdb.incr( tsdb.models.project_total_received_discarded, project, timestamp=to_datetime(start_time) if start_time is not None else None, ) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def set_assemble_status(project, checksum, state, detail=None): cache_key = 'assemble-status:%s' % _get_idempotency_id( project, checksum) # If the state is okay we actually clear it from the cache because in # that case a project dsym file was created. if state == ChunkFileState.OK: default_cache.delete(cache_key) else: default_cache.set(cache_key, (state, detail), 300)
def _process(self, create_task, task_name): task_id = default_cache.get(self.task_id_cache_key) json_response = None with self.sess: try: if task_id: # Processing has already started and we need to poll # symbolicator for an update. This in turn may put us back into # the queue. json_response = self.sess.query_task(task_id) if json_response is None: # This is a new task, so we compute all request parameters # (potentially expensive if we need to pull minidumps), and then # upload all information to symbolicator. It will likely not # have a response ready immediately, so we start polling after # some timeout. json_response = create_task() except ServiceUnavailable: # 503 can indicate that symbolicator is restarting. Wait for a # reboot, then try again. This overrides the default behavior of # retrying after just a second. # # If there is no response attached, it's a connection error. raise RetrySymbolication( retry_after=settings.SYMBOLICATOR_MAX_RETRY_AFTER) metrics.incr( "events.symbolicator.response", tags={ "response": json_response.get("status") or "null", "task_name": task_name }, ) # Symbolication is still in progress. Bail out and try again # after some timeout. Symbolicator keeps the response for the # first one to poll it. if json_response["status"] == "pending": default_cache.set(self.task_id_cache_key, json_response["request_id"], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication( retry_after=json_response["retry_after"]) else: # Once we arrive here, we are done processing. Clean up the # task id from the cache. default_cache.delete(self.task_id_cache_key) metrics.timing("events.symbolicator.response.completed.size", len(json.dumps(json_response))) reverse_source_aliases(json_response) redact_internal_sources(json_response) return json_response
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project = data.pop('project') delete_raw_event(project, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) except HashDiscarded as exc: info_logger.info('discarded.hash', extra={ 'project_id': project.id, 'message': exc.message, }) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def _process(self, create_task): task_id = default_cache.get(self.task_id_cache_key) json = None with self.sess: try: if task_id: # Processing has already started and we need to poll # symbolicator for an update. This in turn may put us back into # the queue. json = self.sess.query_task(task_id) if json is None: # This is a new task, so we compute all request parameters # (potentially expensive if we need to pull minidumps), and then # upload all information to symbolicator. It will likely not # have a response ready immediately, so we start polling after # some timeout. json = create_task() except ServiceUnavailable: # 503 can indicate that symbolicator is restarting. Wait for a # reboot, then try again. This overrides the default behavior of # retrying after just a second. # # If there is no response attached, it's a connection error. raise RetrySymbolication(retry_after=10) metrics.incr('events.symbolicator.response', tags={ 'response': json.get('status') or 'null', 'project_id': self.sess.project_id, }) # Symbolication is still in progress. Bail out and try again # after some timeout. Symbolicator keeps the response for the # first one to poll it. if json['status'] == 'pending': default_cache.set( self.task_id_cache_key, json['request_id'], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication(retry_after=json['retry_after']) else: # Once we arrive here, we are done processing. Clean up the # task id from the cache. default_cache.delete(self.task_id_cache_key) return json
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import tsdb if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'}) return project = data.pop('project') delete_raw_event(project, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) except HashDiscarded as exc: # TODO(jess): remove this before it goes out to a wider audience info_logger.info( 'discarded.hash', extra={ 'project_id': project, 'description': exc.message, } ) tsdb.incr(tsdb.models.project_total_received_discarded, project, timestamp=start_time) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing( 'events.time-to-process', time() - start_time, instance=data['platform'])
def save_event(cache_key=None, data=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import EventManager if cache_key: data = default_cache.get(cache_key) if data is None: return project = data.pop('project') try: manager = EventManager(data) manager.save(project) finally: if cache_key: default_cache.delete(cache_key)
def save_event(cache_key=None, data=None, start_time=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import EventManager if cache_key: data = default_cache.get(cache_key) if data is None: return project = data.pop('project') try: manager = EventManager(data) manager.save(project) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time)
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager, track_outcome from sentry import quotas from sentry.models import ProjectKey if cache_key and data is None: data = default_cache.get(cache_key) if data is not None: data = CanonicalKeyDict(data) if event_id is None and data is not None: event_id = data['event_id'] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop('project') key_id = None if data is None else data.get('key_id') timestamp = to_datetime(start_time) if start_time is not None else None delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }, skip_internal=False) return with configure_scope() as scope: scope.set_tag("project", project_id) event = None try: manager = EventManager(data) event = manager.save(project_id, assume_normalized=True) # Always load attachments from the cache so we can later prune them. # Only save them if the event-attachments feature is active, though. if features.has('organizations:event-attachments', event.project.organization, actor=None): attachments = attachment_cache.get(cache_key) or [] for attachment in attachments: save_attachment(event, attachment) # This is where we can finally say that we have accepted the event. track_outcome(event.project.organization_id, event.project.id, key_id, 'accepted', None, timestamp) except HashDiscarded: project = Project.objects.get_from_cache(id=project_id) reason = FilterStatKeys.DISCARDED_HASH project_key = None try: if key_id is not None: project_key = ProjectKey.objects.get_from_cache(id=key_id) except ProjectKey.DoesNotExist: pass quotas.refund(project, key=project_key, timestamp=start_time) track_outcome(project.organization_id, project_id, key_id, 'filtered', reason, timestamp) finally: if cache_key: default_cache.delete(cache_key) # For the unlikely case that we did not manage to persist the # event we also delete the key always. if event is None or \ features.has('organizations:event-attachments', event.project.organization, actor=None): attachment_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None, reprocessing_rev=None): """If processing failed we put the original data from the cache into a raw event. Returns `True` if a failed event was inserted """ reprocessing_active = ProjectOption.objects.get_value( project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT) # In case there is reprocessing active but the current reprocessing # revision is already different than when we started, we want to # immediately retry the event. This resolves the problem when # otherwise a concurrent change of debug symbols might leave a # reprocessing issue stuck in the project forever. if reprocessing_active and \ reprocessing.get_reprocessing_revision(project_id, cached=False) != \ reprocessing_rev: raise RetryProcessing() # The first time we encounter a failed event and the hint was cleared # we send a notification. sent_notification = ProjectOption.objects.get_value( project_id, 'sentry:sent_failed_event_hint', False) if not sent_notification: project = Project.objects.get_from_cache(id=project_id) Activity.objects.create( type=Activity.NEW_PROCESSING_ISSUES, project=project, datetime=to_datetime(start_time), data={ 'reprocessing_active': reprocessing_active, 'issues': issues }, ).send_notification() ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True) # If reprocessing is not active we bail now without creating the # processing issues if not reprocessing_active: return False # We need to get the original data here instead of passing the data in # from the last processing step because we do not want any # modifications to take place. delete_raw_event(project_id, event_id) data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'raw' }, skip_internal=False) error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key}) return True data = CanonicalKeyDict(data) from sentry.models import RawEvent, ProcessingIssue raw_event = RawEvent.objects.create( project_id=project_id, event_id=event_id, datetime=datetime.utcfromtimestamp( data['timestamp']).replace(tzinfo=timezone.utc), data=data) for issue in issues: ProcessingIssue.objects.record_processing_issue( raw_event=raw_event, scope=issue['scope'], object=issue['object'], type=issue['type'], data=issue['data'], ) default_cache.delete(cache_key) return True
def run_symbolicator(stacktraces, modules, project, arch, signal, request_id_cache_key): internal_url_prefix = options.get('system.internal-url-prefix') \ or options.get('system.url-prefix') assert internal_url_prefix sentry_source_url = '%s%s' % ( internal_url_prefix.rstrip('/'), reverse('sentry-api-0-dsym-files', kwargs={ 'organization_slug': project.organization.slug, 'project_slug': project.slug })) symbolicator_options = options.get('symbolicator.options') base_url = symbolicator_options['url'].rstrip('/') assert base_url project_id = six.text_type(project.id) request_id = default_cache.get(request_id_cache_key) sess = Session() attempts = 0 wait = 0.5 with sess: while 1: try: if request_id: rv = _poll_symbolication_task(sess=sess, base_url=base_url, request_id=request_id) else: rv = _create_symbolication_task( sess=sess, base_url=base_url, project_id=project_id, sentry_source_url=sentry_source_url, signal=signal, stacktraces=stacktraces, modules=modules) metrics.incr('events.symbolicator.status.%s' % rv.status_code, tags={'project_id': project_id}) if rv.status_code == 404 and request_id: default_cache.delete(request_id_cache_key) request_id = None continue elif rv.status_code == 503: raise RetrySymbolication(retry_after=10) rv.raise_for_status() json = rv.json() metrics.incr('events.symbolicator.response.%s' % json['status'], tags={'project_id': project_id}) if json['status'] == 'pending': default_cache.set(request_id_cache_key, json['request_id'], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication(retry_after=json['retry_after']) elif json['status'] == 'completed': default_cache.delete(request_id_cache_key) return rv.json() else: logger.error("Unexpected status: %s", json['status']) default_cache.delete(request_id_cache_key) return except (IOError, RequestException): attempts += 1 if attempts > MAX_ATTEMPTS: logger.error('Failed to contact symbolicator', exc_info=True) default_cache.delete(request_id_cache_key) return time.sleep(wait) wait *= 2.0
def save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas, tsdb from sentry.models import ProjectKey if cache_key: data = default_cache.get(cache_key) if data is not None: data = CanonicalKeyDict(data) if event_id is None and data is not None: event_id = data['event_id'] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop('project') delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return Raven.tags_context({ 'project': project_id, }) try: manager = EventManager(data) event = manager.save(project_id) # Always load attachments from the cache so we can later prune them. # Only save them if the event-attachments feature is active, though. if features.has('organizations:event-attachments', event.project.organization, actor=None): attachments = attachment_cache.get(cache_key) or [] for attachment in attachments: save_attachment(event, attachment) except HashDiscarded: increment_list = [ (tsdb.models.project_total_received_discarded, project_id), ] try: project = Project.objects.get_from_cache(id=project_id) except Project.DoesNotExist: pass else: increment_list.extend([ (tsdb.models.project_total_blacklisted, project.id), (tsdb.models.organization_total_blacklisted, project.organization_id), ]) project_key = None if data.get('key_id') is not None: try: project_key = ProjectKey.objects.get_from_cache( id=data['key_id']) except ProjectKey.DoesNotExist: pass else: increment_list.append( (tsdb.models.key_total_blacklisted, project_key.id)) quotas.refund( project, key=project_key, timestamp=start_time, ) tsdb.incr_multi( increment_list, timestamp=to_datetime(start_time) if start_time is not None else None, ) finally: if cache_key: default_cache.delete(cache_key) attachment_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def post(self, request): """ Registers a Relay ````````````````` Registers the relay with the sentry installation. If a relay boots it will always attempt to invoke this endpoint. """ try: json_data = json.loads(request.body) except ValueError: return Response({ 'detail': 'No valid json body', }, status=status.HTTP_400_BAD_REQUEST) serializer = RelayRegisterResponseSerializer(data=json_data) if not serializer.is_valid(): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) sig = get_header_relay_signature(request) if not sig: return Response({ 'detail': 'Missing relay signature', }, status=status.HTTP_400_BAD_REQUEST) relay_id = six.text_type(get_register_response_relay_id(request.body)) if relay_id != get_header_relay_id(request): return Response({ 'detail': 'relay_id in payload did not match header', }, status=status.HTTP_400_BAD_REQUEST) params = default_cache.get('relay-auth:%s' % relay_id) if params is None: return Response({ 'detail': 'Challenge expired' }, status=status.HTTP_401_UNAUTHORIZED) key = PublicKey.parse(params['public_key']) try: validate_register_response(key, request.body, sig) except Exception as exc: return Response({ 'detail': str(exc).splitlines()[0], }, status=status.HTTP_400_BAD_REQUEST) is_internal = is_internal_relay(request, params['public_key']) try: relay = Relay.objects.get(relay_id=relay_id) except Relay.DoesNotExist: relay = Relay.objects.create( relay_id=relay_id, public_key=params['public_key'], is_internal=is_internal ) else: relay.last_seen = timezone.now() relay.is_internal = is_internal relay.save() default_cache.delete('relay-auth:%s' % relay_id) return Response(serialize({ 'relay_id': relay.relay_id, }))
def run_symbolicator(stacktraces, modules, project, arch, signal, request_id_cache_key): symbolicator_options = options.get('symbolicator.options') base_url = symbolicator_options['url'].rstrip('/') assert base_url project_id = six.text_type(project.id) request_id = default_cache.get(request_id_cache_key) sess = Session() # Will be set lazily when a symbolicator request is fired sources = None attempts = 0 wait = 0.5 with sess: while True: try: if request_id: rv = _poll_symbolication_task(sess=sess, base_url=base_url, request_id=request_id) else: if sources is None: sources = get_sources_for_project(project) rv = _create_symbolication_task(sess=sess, base_url=base_url, project_id=project_id, sources=sources, signal=signal, stacktraces=stacktraces, modules=modules) metrics.incr('events.symbolicator.status_code', tags={ 'status_code': rv.status_code, 'project_id': project_id, }) if rv.status_code == 404 and request_id: default_cache.delete(request_id_cache_key) request_id = None continue elif rv.status_code == 503: raise RetrySymbolication(retry_after=10) rv.raise_for_status() json = rv.json() metrics.incr('events.symbolicator.response', tags={ 'response': json['status'], 'project_id': project_id, }) if json['status'] == 'pending': default_cache.set(request_id_cache_key, json['request_id'], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication(retry_after=json['retry_after']) elif json['status'] == 'completed': default_cache.delete(request_id_cache_key) return rv.json() else: logger.error("Unexpected status: %s", json['status']) default_cache.delete(request_id_cache_key) return except (IOError, RequestException): attempts += 1 if attempts > MAX_ATTEMPTS: logger.error('Failed to contact symbolicator', exc_info=True) default_cache.delete(request_id_cache_key) return time.sleep(wait) wait *= 2.0
def save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas, tsdb from sentry.models import ProjectKey if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop('project') delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'}) return Raven.tags_context({ 'project': project_id, }) try: manager = EventManager(data) manager.save(project_id) except HashDiscarded: increment_list = [ (tsdb.models.project_total_received_discarded, project_id), ] try: project = Project.objects.get_from_cache(id=project_id) except Project.DoesNotExist: pass else: increment_list.extend([ (tsdb.models.project_total_blacklisted, project.id), (tsdb.models.organization_total_blacklisted, project.organization_id), ]) project_key = None if data.get('key_id') is not None: try: project_key = ProjectKey.objects.get_from_cache(id=data['key_id']) except ProjectKey.DoesNotExist: pass else: increment_list.append((tsdb.models.key_total_blacklisted, project_key.id)) quotas.refund( project, key=project_key, timestamp=start_time, ) tsdb.incr_multi( increment_list, timestamp=to_datetime(start_time) if start_time is not None else None, ) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing( 'events.time-to-process', time() - start_time, instance=data['platform'])
def _do_save_event( cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs ): """ Saves an event to the database. """ set_current_project(project_id) from sentry.event_manager import EventManager, HashDiscarded event_type = "none" if cache_key and data is None: with metrics.timer("tasks.store.do_save_event.get_cache") as metric_tags: data = default_cache.get(cache_key) if data is not None: metric_tags["event_type"] = event_type = data.get("type") or "none" with metrics.global_tags(event_type=event_type): if data is not None: data = CanonicalKeyDict(data) if event_id is None and data is not None: event_id = data["event_id"] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop("project") set_current_project(project_id) # We only need to delete raw events for events that support # reprocessing. If the data cannot be found we want to assume # that we need to delete the raw event. if not data or reprocessing.event_supports_reprocessing(data): with metrics.timer("tasks.store.do_save_event.delete_raw_event"): delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr( "events.failed", tags={"reason": "cache", "stage": "post"}, skip_internal=False ) return event = None try: with metrics.timer("tasks.store.do_save_event.event_manager.save"): manager = EventManager(data) # event.project.organization is populated after this statement. event = manager.save( project_id, assume_normalized=True, start_time=start_time, cache_key=cache_key ) except HashDiscarded: pass finally: if cache_key: with metrics.timer("tasks.store.do_save_event.delete_cache"): default_cache.delete(cache_key) with metrics.timer("tasks.store.do_save_event.delete_attachment_cache"): # For the unlikely case that we did not manage to persist the # event we also delete the key always. if event is None or features.has( "organizations:event-attachments", event.project.organization, actor=None ): attachment_cache.delete(cache_key) if start_time: metrics.timing( "events.time-to-process", time() - start_time, instance=data["platform"] )
def run_symbolicator(project, request_id_cache_key, create_task=create_payload_task, **kwargs): symbolicator_options = options.get('symbolicator.options') base_url = symbolicator_options['url'].rstrip('/') assert base_url project_id = six.text_type(project.id) request_id = default_cache.get(request_id_cache_key) sess = Session() # Will be set lazily when a symbolicator request is fired sources = None attempts = 0 wait = 0.5 with sess: while True: try: if request_id: rv = _poll_symbolication_task( sess=sess, base_url=base_url, request_id=request_id, project_id=project_id, ) else: if sources is None: sources = get_sources_for_project(project) rv = create_task( sess=sess, base_url=base_url, project_id=project_id, sources=sources, **kwargs ) metrics.incr('events.symbolicator.status_code', tags={ 'status_code': rv.status_code, 'project_id': project_id, }) if rv.status_code == 404 and request_id: default_cache.delete(request_id_cache_key) request_id = None continue elif rv.status_code == 503: raise RetrySymbolication(retry_after=10) rv.raise_for_status() json = rv.json() metrics.incr('events.symbolicator.response', tags={ 'response': json['status'], 'project_id': project_id, }) if json['status'] == 'pending': default_cache.set( request_id_cache_key, json['request_id'], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication(retry_after=json['retry_after']) elif json['status'] == 'completed': default_cache.delete(request_id_cache_key) return rv.json() else: logger.error("Unexpected status: %s", json['status']) default_cache.delete(request_id_cache_key) return except (IOError, RequestException): attempts += 1 if attempts > MAX_ATTEMPTS: logger.error('Failed to contact symbolicator', exc_info=True) default_cache.delete(request_id_cache_key) return time.sleep(wait) wait *= 2.0
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas from sentry.models import ProjectKey from sentry.utils.outcomes import Outcome, track_outcome if cache_key and data is None: data = default_cache.get(cache_key) if data is not None: data = CanonicalKeyDict(data) if event_id is None and data is not None: event_id = data['event_id'] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop('project') key_id = None if data is None else data.get('key_id') if key_id is not None: key_id = int(key_id) timestamp = to_datetime(start_time) if start_time is not None else None delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr( 'events.failed', tags={ 'reason': 'cache', 'stage': 'post'}, skip_internal=False) return with configure_scope() as scope: scope.set_tag("project", project_id) event = None try: manager = EventManager(data) event = manager.save(project_id, assume_normalized=True) # Always load attachments from the cache so we can later prune them. # Only save them if the event-attachments feature is active, though. if features.has('organizations:event-attachments', event.project.organization, actor=None): attachments = attachment_cache.get(cache_key) or [] for attachment in attachments: save_attachment(event, attachment) # This is where we can finally say that we have accepted the event. track_outcome( event.project.organization_id, event.project.id, key_id, Outcome.ACCEPTED, None, timestamp, event_id ) except HashDiscarded: project = Project.objects.get_from_cache(id=project_id) reason = FilterStatKeys.DISCARDED_HASH project_key = None try: if key_id is not None: project_key = ProjectKey.objects.get_from_cache(id=key_id) except ProjectKey.DoesNotExist: pass quotas.refund(project, key=project_key, timestamp=start_time) track_outcome( project.organization_id, project_id, key_id, Outcome.FILTERED, reason, timestamp, event_id ) finally: if cache_key: default_cache.delete(cache_key) # For the unlikely case that we did not manage to persist the # event we also delete the key always. if event is None or \ features.has('organizations:event-attachments', event.project.organization, actor=None): attachment_cache.delete(cache_key) if start_time: metrics.timing( 'events.time-to-process', time() - start_time, instance=data['platform'])
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas, tsdb from sentry.models import ProjectKey if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project_id = data.pop('project') delete_raw_event(project_id, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project_id, }) try: manager = EventManager(data) manager.save(project_id) except HashDiscarded: tsdb.incr( tsdb.models.project_total_received_discarded, project_id, timestamp=to_datetime(start_time) if start_time is not None else None, ) try: project = Project.objects.get_from_cache(id=project_id) except Project.DoesNotExist: pass else: project_key = None if data.get('key_id') is not None: try: project_key = ProjectKey.objects.get_from_cache( id=data['key_id']) except ProjectKey.DoesNotExist: pass quotas.refund( project, key=project_key, timestamp=start_time, ) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def create_failed_event(cache_key, project_id, issues, event_id, start_time=None, reprocessing_rev=None): """If processing failed we put the original data from the cache into a raw event. Returns `True` if a failed event was inserted """ reprocessing_active = ProjectOption.objects.get_value( project_id, 'sentry:reprocessing_active', REPROCESSING_DEFAULT ) # In case there is reprocessing active but the current reprocessing # revision is already different than when we started, we want to # immediately retry the event. This resolves the problem when # otherwise a concurrent change of debug symbols might leave a # reprocessing issue stuck in the project forever. if reprocessing_active and \ reprocessing.get_reprocessing_revision(project_id, cached=False) != \ reprocessing_rev: raise RetryProcessing() # The first time we encounter a failed event and the hint was cleared # we send a notification. sent_notification = ProjectOption.objects.get_value( project_id, 'sentry:sent_failed_event_hint', False ) if not sent_notification: project = Project.objects.get_from_cache(id=project_id) Activity.objects.create( type=Activity.NEW_PROCESSING_ISSUES, project=project, datetime=to_datetime(start_time), data={'reprocessing_active': reprocessing_active, 'issues': issues}, ).send_notification() ProjectOption.objects.set_value(project, 'sentry:sent_failed_event_hint', True) # If reprocessing is not active we bail now without creating the # processing issues if not reprocessing_active: return False # We need to get the original data here instead of passing the data in # from the last processing step because we do not want any # modifications to take place. delete_raw_event(project_id, event_id) data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'raw'}, skip_internal=False) error_logger.error('process.failed_raw.empty', extra={'cache_key': cache_key}) return True data = CanonicalKeyDict(data) from sentry.models import RawEvent, ProcessingIssue raw_event = RawEvent.objects.create( project_id=project_id, event_id=event_id, datetime=datetime.utcfromtimestamp(data['timestamp']).replace(tzinfo=timezone.utc), data=data ) for issue in issues: ProcessingIssue.objects.record_processing_issue( raw_event=raw_event, scope=issue['scope'], object=issue['object'], type=issue['type'], data=issue['data'], ) default_cache.delete(cache_key) return True
def _do_save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas from sentry.models import ProjectKey from sentry.utils.outcomes import Outcome, track_outcome from sentry.ingest.outcomes_consumer import mark_signal_sent event_type = "none" if cache_key and data is None: with metrics.timer( "tasks.store.do_save_event.get_cache") as metric_tags: data = default_cache.get(cache_key) if data is not None: metric_tags["event_type"] = event_type = data.get( "type") or "none" with metrics.global_tags(event_type=event_type): if data is not None: data = CanonicalKeyDict(data) if event_id is None and data is not None: event_id = data["event_id"] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop("project") key_id = None if data is None else data.get("key_id") if key_id is not None: key_id = int(key_id) timestamp = to_datetime(start_time) if start_time is not None else None # We only need to delete raw events for events that support # reprocessing. If the data cannot be found we want to assume # that we need to delete the raw event. if not data or reprocessing.event_supports_reprocessing(data): with metrics.timer("tasks.store.do_save_event.delete_raw_event"): delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr("events.failed", tags={ "reason": "cache", "stage": "post" }, skip_internal=False) return with configure_scope() as scope: scope.set_tag("project", project_id) event = None try: with metrics.timer("tasks.store.do_save_event.event_manager.save"): manager = EventManager(data) # event.project.organization is populated after this statement. event = manager.save(project_id, assume_normalized=True, cache_key=cache_key) with metrics.timer("tasks.store.do_save_event.track_outcome"): # This is where we can finally say that we have accepted the event. track_outcome( event.project.organization_id, event.project.id, key_id, Outcome.ACCEPTED, None, timestamp, event_id, ) except HashDiscarded: project = Project.objects.get_from_cache(id=project_id) reason = FilterStatKeys.DISCARDED_HASH project_key = None try: if key_id is not None: project_key = ProjectKey.objects.get_from_cache(id=key_id) except ProjectKey.DoesNotExist: pass quotas.refund(project, key=project_key, timestamp=start_time) # There is no signal supposed to be sent for this particular # outcome-reason combination. Prevent the outcome consumer from # emitting it for now. # # XXX(markus): Revisit decision about signals once outcomes consumer is stable. mark_signal_sent(project_id, event_id) track_outcome( project.organization_id, project_id, key_id, Outcome.FILTERED, reason, timestamp, event_id, ) finally: if cache_key: with metrics.timer("tasks.store.do_save_event.delete_cache"): default_cache.delete(cache_key) with metrics.timer( "tasks.store.do_save_event.delete_attachment_cache"): # For the unlikely case that we did not manage to persist the # event we also delete the key always. if event is None or features.has( "organizations:event-attachments", event.project.organization, actor=None): attachment_cache.delete(cache_key) if start_time: metrics.timing("events.time-to-process", time() - start_time, instance=data["platform"])