def record_affected_user(event, **kwargs): from sentry.models import EventUser, Group Raven.tags_context({ 'project': event.project_id, }) user_data = event.data.get('sentry.interfaces.User', event.data.get('user')) if not user_data: logger.info('No user data found for event_id=%s', event.event_id) return euser = EventUser( project=event.project, ident=user_data.get('id'), email=user_data.get('email'), username=user_data.get('username'), ip_address=user_data.get('ip_address'), ) if not euser.tag_value: # no ident, bail logger.info('No identifying value found for user on event_id=%s', event.event_id) return try: with transaction.atomic(using=router.db_for_write(EventUser)): euser.save() except IntegrityError: pass Group.objects.add_tags(event.group, [ ('sentry:user', euser.tag_value) ])
def index_event_tags(organization_id, project_id, event_id, tags, group_id, environment_id, date_added=None, **kwargs): from sentry import tagstore Raven.tags_context({ 'project': project_id, }) create_event_tags_kwargs = {} if date_added is not None: create_event_tags_kwargs['date_added'] = date_added metrics.timing('tagstore.tags_per_event', len(tags), tags={ 'organization_id': organization_id, }) tagstore.create_event_tags(project_id=project_id, group_id=group_id, environment_id=environment_id, event_id=event_id, tags=tags, **create_event_tags_kwargs)
def _do_preprocess_event(cache_key, data, start_time, event_id, process_event): if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}) error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key}) return project = data['project'] Raven.tags_context({ 'project': project, }) if should_process(data): process_event.delay(cache_key=cache_key, start_time=start_time, event_id=event_id) return # If we get here, that means the event had no preprocessing needed to be done # so we can jump directly to save_event if cache_key: data = None save_event.delay(cache_key=cache_key, data=data, start_time=start_time, event_id=event_id)
def index_event_tags(organization_id, project_id, event_id, tags, group_id, environment_id, **kwargs): from sentry import tagstore Raven.tags_context({ 'project': project_id, }) tag_ids = [] for key, value in tags: tagkey, _ = tagstore.get_or_create_tag_key(project_id, environment_id, key) # TODO(brett): optimization to hand `get_or_create_tag_value` the key_id # so it doesn't have to hit the database for something we just had on hand tagvalue, _ = tagstore.get_or_create_tag_value(project_id, environment_id, key, value) tag_ids.append((tagkey.id, tagvalue.id)) tagstore.create_event_tags( project_id=project_id, group_id=group_id, environment_id=environment_id, event_id=event_id, tags=tag_ids, )
def index_event_tags(organization_id, project_id, event_id, tags, group_id=None, **kwargs): from sentry.models import EventTag, Project, TagKey, TagValue Raven.tags_context({ 'project': project_id, }) for key, value in tags: tagkey, _ = TagKey.objects.get_or_create( project=Project(id=project_id), key=key, ) tagvalue, _ = TagValue.objects.get_or_create( project=Project(id=project_id, organization_id=organization_id), key=key, value=value, ) try: # handle replaying of this task with transaction.atomic(): EventTag.objects.create( project_id=project_id, group_id=group_id, event_id=event_id, key_id=tagkey.id, value_id=tagvalue.id, ) except IntegrityError: pass
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): # A CSP report is sent as a POST request with no Origin or Referer # header. What we're left with is a 'document-uri' key which is # inside of the JSON body of the request. This 'document-uri' value # should be treated as an origin check since it refers to the page # that triggered the report. The Content-Type is supposed to be # `application/csp-report`, but FireFox sends it as `application/json`. if request.method != 'POST': return HttpResponseNotAllowed(['POST']) if request.META.get('CONTENT_TYPE') not in self.content_types: raise APIError('Invalid Content-Type') request.user = AnonymousUser() project = self._get_project_from_id(project_id) helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) # This is yanking the auth from the querystring since it's not # in the POST body. This means we expect a `sentry_key` and # `sentry_version` to be set in querystring auth = helper.auth_from_request(request) key = helper.project_key_from_auth(auth) if key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) return super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, **kwargs )
def post_process_group(event, is_new, is_regression, is_sample, **kwargs): """ Fires post processing hooks for a group. """ from sentry.models import Project from sentry.rules.processor import RuleProcessor project_id = event.group.project_id Raven.tags_context({ 'project': project_id, }) project = Project.objects.get_from_cache(id=project_id) _capture_stats(event, is_new) rp = RuleProcessor(event, is_new, is_regression, is_sample) # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): safe_execute(callback, event, futures) for plugin in plugins.for_project(project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression, is_sample=is_sample, )
def index_event_tags(organization_id, project_id, event_id, tags, group_id=None, **kwargs): from sentry.models import EventTag, Project, TagKey, TagValue Raven.tags_context({ 'project': project_id, }) for key, value in tags: tagkey, _ = TagKey.objects.get_or_create( project=Project(id=project_id), key=key, ) tagvalue, _ = TagValue.objects.get_or_create( project=Project(id=project_id, organization_id=organization_id), key=key, value=value, ) try: # handle replaying of this task with transaction.atomic(): EventTag.objects.create( project_id=project_id, group_id=group_id, event_id=event_id, key_id=tagkey.id, value_id=tagvalue.id, ) except IntegrityError: pass
def _do_preprocess_event(cache_key, data, start_time, event_id, process_event): if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}) error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key}) return data = CanonicalKeyDict(data) project = data['project'] Raven.tags_context({ 'project': project, }) if should_process(data): process_event.delay(cache_key=cache_key, start_time=start_time, event_id=event_id) return # If we get here, that means the event had no preprocessing needed to be done # so we can jump directly to save_event if cache_key: data = None save_event.delay(cache_key=cache_key, data=data, start_time=start_time, event_id=event_id, project_id=project)
def _do_preprocess_event(cache_key, data, start_time, event_id, process_event): if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}) error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key}) return project_id = data['project'] Raven.tags_context({ 'project': project_id, }) if should_process(data): # save another version of data for some projects to generate # preprocessing hash that won't be modified by pipeline hash_cache.set(get_raw_cache_key(project_id, data['event_id']), data) process_event.delay(cache_key=cache_key, start_time=start_time, event_id=event_id) return # If we get here, that means the event had no preprocessing needed to be done # so we can jump directly to save_event if cache_key: data = None save_event.delay(cache_key=cache_key, data=data, start_time=start_time, event_id=event_id)
def _parse_header(self, request, project): try: auth_vars = extract_auth_vars(request) except (IndexError, ValueError): raise APIError('Invalid auth header') if not auth_vars: raise APIError( 'Client/server version mismatch: Unsupported client') server_version = auth_vars.get('sentry_version', '1.0') client = auth_vars.get('sentry_client', request.META.get('HTTP_USER_AGENT')) Raven.tags_context({'client': client}) Raven.tags_context({'protocol': server_version}) if server_version not in PROTOCOL_VERSIONS: raise APIError( 'Client/server version mismatch: Unsupported protocol version (%s)' % server_version) if not client: raise APIError( 'Client request error: Missing client version identifier') return auth_vars
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs): from sentry.plugins import plugins if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}) logger.error('Data not available in preprocess_event (cache_key=%s)', cache_key) return project = data['project'] Raven.tags_context({ 'project': project, }) # TODO(dcramer): ideally we would know if data changed by default has_changed = False for plugin in plugins.all(version=2): processors = safe_execute(plugin.get_event_preprocessors, _with_transaction=False) for processor in (processors or ()): result = safe_execute(processor, data) if result: data = result has_changed = True assert data['project'] == project, 'Project cannot be mutated by preprocessor' if has_changed and cache_key: default_cache.set(cache_key, data, 3600) if cache_key: data = None save_event.delay(cache_key=cache_key, data=data, start_time=start_time)
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): # A CSP report is sent as a POST request with no Origin or Referer # header. What we're left with is a 'document-uri' key which is # inside of the JSON body of the request. This 'document-uri' value # should be treated as an origin check since it refers to the page # that triggered the report. The Content-Type is supposed to be # `application/csp-report`, but FireFox sends it as `application/json`. if request.method != 'POST': return HttpResponseNotAllowed(['POST']) if request.META.get('CONTENT_TYPE') not in self.content_types: raise APIError('Invalid Content-Type') request.user = AnonymousUser() project = self._get_project_from_id(project_id) helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) # This is yanking the auth from the querystring since it's not # in the POST body. This means we expect a `sentry_key` and # `sentry_version` to be set in querystring auth = helper.auth_from_request(request) key = helper.project_key_from_auth(auth) if key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) return super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, **kwargs )
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs): from sentry.plugins import plugins if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}) error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key}) return project = data['project'] Raven.tags_context({ 'project': project, }) # Iterate over all plugins looking for processors based on the input data # plugins should yield a processor function only if it actually can operate # on the input data, otherwise it should yield nothing for plugin in plugins.all(version=2): processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False) for processor in (processors or ()): # On the first processor found, we just defer to the process_event # queue to handle the actual work. process_event.delay(cache_key=cache_key, start_time=start_time) return # If we get here, that means the event had no preprocessing needed to be done # so we can jump directly to save_event if cache_key: data = None save_event.delay(cache_key=cache_key, data=data, start_time=start_time)
def save_event(cache_key=None, data=None, start_time=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import EventManager if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project = data.pop('project') Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def index_event_tags(organization_id, project_id, event_id, tags, group_id=None, **kwargs): from sentry import tagstore from sentry.models import EventTag Raven.tags_context({ 'project': project_id, }) for key, value in tags: tagkey, _ = tagstore.get_or_create_tag_key(project_id, key) tagvalue, _ = tagstore.get_or_create_tag_value(project_id, key, value) try: # handle replaying of this task with transaction.atomic(): EventTag.objects.create( project_id=project_id, group_id=group_id, event_id=event_id, key_id=tagkey.id, value_id=tagvalue.id, ) except IntegrityError: pass
def preprocess_event(cache_key=None, data=None, start_time=None, **kwargs): from sentry.plugins import plugins if cache_key: data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'pre'}) error_logger.error('preprocess.failed.empty', extra={'cache_key': cache_key}) return project = data['project'] Raven.tags_context({ 'project': project, }) # Iterate over all plugins looking for processors based on the input data # plugins should yield a processor function only if it actually can operate # on the input data, otherwise it should yield nothing for plugin in plugins.all(version=2): processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False) for processor in (processors or ()): # On the first processor found, we just defer to the process_event # queue to handle the actual work. process_event.delay(cache_key=cache_key, start_time=start_time) return # If we get here, that means the event had no preprocessing needed to be done # so we can jump directly to save_event if cache_key: data = None save_event.delay(cache_key=cache_key, data=data, start_time=start_time)
def index_event_tags(organization_id, project_id, event_id, tags, group_id, environment_id, **kwargs): from sentry import tagstore Raven.tags_context({ 'project': project_id, }) tag_ids = [] for key, value in tags: tagkey, _ = tagstore.get_or_create_tag_key(project_id, environment_id, key) tagvalue, _ = tagstore.get_or_create_tag_value(project_id, environment_id, key, value, key_id=tagkey.id) tag_ids.append((tagkey.id, tagvalue.id)) tagstore.create_event_tags( project_id=project_id, group_id=group_id, environment_id=environment_id, event_id=event_id, tags=tag_ids, )
def record_affected_user(event, **kwargs): from sentry.models import EventUser, Group Raven.tags_context({ 'project': event.project_id, }) user_data = event.data.get('sentry.interfaces.User', event.data.get('user')) if not user_data: logger.info('No user data found for event_id=%s', event.event_id) return euser = EventUser( project=event.project, ident=user_data.get('id'), email=user_data.get('email'), username=user_data.get('username'), ip_address=user_data.get('ip_address'), ) if not euser.tag_value: # no ident, bail logger.info('No identifying value found for user on event_id=%s', event.event_id) return try: with transaction.atomic(using=router.db_for_write(EventUser)): euser.save() except IntegrityError: pass Group.objects.add_tags(event.group, [('sentry:user', euser.tag_value)])
def index_event_tags(organization_id, project_id, event_id, tags, group_id, environment_id, date_added=None, **kwargs): from sentry import tagstore Raven.tags_context({ 'project': project_id, }) create_event_tags_kwargs = {} if date_added is not None: create_event_tags_kwargs['date_added'] = date_added metrics.timing( 'tagstore.tags_per_event', len(tags), tags={ 'organization_id': organization_id, } ) tagstore.create_event_tags( project_id=project_id, group_id=group_id, environment_id=environment_id, event_id=event_id, tags=tags, **create_event_tags_kwargs )
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import EventManager if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'}) return project = data.pop('project') delete_raw_event(project, event_id) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): tsdb.incr(tsdb.models.project_total_received_cors, project.id) raise APIForbidden('Invalid origin: %s' % (origin, )) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) key = helper.project_key_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: project = Project.objects.get_from_cache(id=key.project_id) helper.context.bind_project(project) elif key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) # Explicitly bind Organization so we don't implicitly query it later # this just allows us to comfortably assure that `project.organization` is safe. # This also allows us to pull the object from cache, instead of being # implicitly fetched from database. project.organization = Organization.objects.get_from_cache( id=project.organization_id) response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, **kwargs ) if origin: if origin == 'null': # If an Origin is `null`, but we got this far, that means # we've gotten past our CORS check for some reason. But the # problem is that we can't return "null" as a valid response # to `Access-Control-Allow-Origin` and we don't have another # value to work with, so just allow '*' since they've gotten # this far. response['Access-Control-Allow-Origin'] = '*' else: response['Access-Control-Allow-Origin'] = origin return response
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): tsdb.incr(tsdb.models.project_total_received_cors, project.id) raise APIForbidden('Invalid origin: %s' % (origin, )) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) key = helper.project_key_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: project = Project.objects.get_from_cache(id=key.project_id) helper.context.bind_project(project) elif key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) # Explicitly bind Organization so we don't implicitly query it later # this just allows us to comfortably assure that `project.organization` is safe. # This also allows us to pull the object from cache, instead of being # implicitly fetched from database. project.organization = Organization.objects.get_from_cache( id=project.organization_id) response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, **kwargs ) if origin: if origin == 'null': # If an Origin is `null`, but we got this far, that means # we've gotten past our CORS check for some reason. But the # problem is that we can't return "null" as a valid response # to `Access-Control-Allow-Origin` and we don't have another # value to work with, so just allow '*' since they've gotten # this far. response['Access-Control-Allow-Origin'] = '*' else: response['Access-Control-Allow-Origin'] = origin return response
def _do_process_event(cache_key, start_time, event_id): from sentry.plugins import plugins data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'process' }) error_logger.error('process.failed.empty', extra={'cache_key': cache_key}) return project = data['project'] Raven.tags_context({ 'project': project, }) has_changed = False # Stacktrace based event processors. These run before anything else. new_data = process_stacktraces(data) if new_data is not None: has_changed = True data = new_data # TODO(dcramer): ideally we would know if data changed by default # Default event processors. for plugin in plugins.all(version=2): processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False) for processor in (processors or ()): result = safe_execute(processor, data) if result: data = result has_changed = True assert data[ 'project'] == project, 'Project cannot be mutated by preprocessor' if has_changed: issues = data.get('processing_issues') if issues and create_failed_event(cache_key, project, list(issues.values()), event_id=event_id, start_time=start_time): return default_cache.set(cache_key, data, 3600) save_event.delay( cache_key=cache_key, data=None, start_time=start_time, event_id=event_id, )
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError("Client must be upgraded for CORS support") if not is_valid_origin(origin, project): raise APIForbidden("Invalid origin: %s" % (origin,)) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == "OPTIONS": response = self.options(request, project) else: auth = self._parse_header(request, helper, project) project_ = helper.project_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise APIError("Unable to identify project") project = project_ helper.context.bind_project(project) elif project_ != project: raise APIError("Two different project were specified") helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) if auth.version != "2.0": if not auth.secret_key: # If we're missing a secret_key, check if we are allowed # to do a CORS request. # If we're missing an Origin/Referrer header entirely, # we only want to support this on GET requests. By allowing # un-authenticated CORS checks for POST, we basially # are obsoleting our need for a secret key entirely. if origin is None and request.method != "GET": raise APIForbidden("Missing required attribute in authentication header: sentry_secret") if not is_valid_origin(origin, project): raise APIForbidden("Missing required Origin or Referer header") response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, **kwargs ) if origin: response["Access-Control-Allow-Origin"] = origin return response
def plugin_post_process_group(plugin_slug, event, **kwargs): """ Fires post processing hooks for a group. """ Raven.tags_context({ 'project': event.project_id, }) plugin = plugins.get(plugin_slug) safe_execute(plugin.post_process, event=event, group=event.group, **kwargs)
def plugin_post_process_group(plugin_slug, event, **kwargs): """ Fires post processing hooks for a group. """ Raven.tags_context({ 'project': event.project_id, }) plugin = plugins.get(plugin_slug) safe_execute(plugin.post_process, event=event, group=event.group, **kwargs)
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import tsdb if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project = data.pop('project') delete_raw_event(project, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) except HashDiscarded as exc: # TODO(jess): remove this before it goes out to a wider audience info_logger.info('discarded.hash', extra={ 'project_id': project, 'description': exc.message, }) tsdb.incr( tsdb.models.project_total_received_discarded, project, timestamp=to_datetime(start_time) if start_time is not None else None, ) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def _wrapped(*args, **kwargs): key = 'jobs.duration' if stat_suffix: instance = '{}.{}'.format(name, stat_suffix(*args, **kwargs)) else: instance = name Raven.tags_context({'task_name': name}) with metrics.timer(key, instance=instance): try: result = func(*args, **kwargs) finally: Raven.context.clear() return result
def post_process_group(event, is_new, is_regression, is_sample, **kwargs): """ Fires post processing hooks for a group. """ # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor # Re-bind Group since we're pickling the whole Event object # which may contain a stale Group. event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id project_id = event.group.project_id Raven.tags_context({ 'project': project_id, }) # Re-bind Project since we're pickling the whole Event object # which may contain a stale Project. event.project = Project.objects.get_from_cache(id=project_id) _capture_stats(event, is_new) # we process snoozes before rules as it might create a regression process_snoozes(event.group) rp = RuleProcessor(event, is_new, is_regression, is_sample) # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): safe_execute(callback, event, futures) for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression, is_sample=is_sample, ) event_processed.send_robust( sender=post_process_group, project=event.project, group=event.group, event=event, )
def post_process_group(event, is_new, is_regression, is_sample, **kwargs): """ Fires post processing hooks for a group. """ # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor # Re-bind Group since we're pickling the whole Event object # which may contain a stale Group. event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id project_id = event.group.project_id Raven.tags_context({ 'project': project_id, }) # Re-bind Project since we're pickling the whole Event object # which may contain a stale Project. event.project = Project.objects.get_from_cache(id=project_id) _capture_stats(event, is_new) # we process snoozes before rules as it might create a regression process_snoozes(event.group) rp = RuleProcessor(event, is_new, is_regression, is_sample) # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): safe_execute(callback, event, futures) for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression, is_sample=is_sample, ) event_processed.send_robust( sender=post_process_group, project=event.project, group=event.group, event=event, )
def _wrapped(*args, **kwargs): key = 'jobs.duration' if stat_suffix: instance = '{}.{}'.format(name, stat_suffix(*args, **kwargs)) else: instance = name Raven.tags_context({'task_name': name}) with metrics.timer(key, instance=instance), \ track_memory_usage('jobs.memory_change', instance=instance): try: result = func(*args, **kwargs) finally: Raven.context.clear() return result
def index_event_tags(organization_id, project_id, event_id, tags, group_id, environment_id, **kwargs): from sentry import tagstore Raven.tags_context({ 'project': project_id, }) tagstore.create_event_tags( project_id=project_id, group_id=group_id, environment_id=environment_id, event_id=event_id, tags=tags, )
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return project = data.pop('project') delete_raw_event(project, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) except HashDiscarded as exc: info_logger.info('discarded.hash', extra={ 'project_id': project.id, 'message': exc.message, }) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): # TODO(ja): Refactor shared code with CspReportView. Especially, look at # the sentry_key override and test it. # A minidump submission as implemented by Breakpad and Crashpad or any # other library following the Mozilla Soccorro protocol is a POST request # without Origin or Referer headers. Therefore, we cannot validate the # origin of the request, but we *can* validate the "prod" key in future. if request.method != 'POST': return HttpResponseNotAllowed(['POST']) content_type = request.META.get('CONTENT_TYPE') # In case of multipart/form-data, the Content-Type header also includes # a boundary. Therefore, we cannot check for an exact match. if content_type is None or not content_type.startswith( self.content_types): raise APIError('Invalid Content-Type') request.user = AnonymousUser() project = self._get_project_from_id(project_id) helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) # This is yanking the auth from the querystring since it's not # in the POST body. This means we expect a `sentry_key` and # `sentry_version` to be set in querystring auth = helper.auth_from_request(request) key = helper.project_key_from_auth(auth) if key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) return super(APIView, self).dispatch(request=request, project=project, auth=auth, helper=helper, key=key, **kwargs)
def _do_process_event(cache_key, start_time, event_id): from sentry.plugins import plugins data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'process'}) error_logger.error('process.failed.empty', extra={'cache_key': cache_key}) return project = data['project'] Raven.tags_context({ 'project': project, }) has_changed = False # Stacktrace based event processors. These run before anything else. new_data = process_stacktraces(data) if new_data is not None: has_changed = True data = new_data # TODO(dcramer): ideally we would know if data changed by default # Default event processors. for plugin in plugins.all(version=2): processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False) for processor in (processors or ()): result = safe_execute(processor, data) if result: data = result has_changed = True assert data['project'] == project, 'Project cannot be mutated by preprocessor' if has_changed: issues = data.get('processing_issues') if issues: create_failed_event(cache_key, project, list(issues.values()), event_id=event_id) return default_cache.set(cache_key, data, 3600) save_event.delay(cache_key=cache_key, data=None, start_time=start_time, event_id=event_id)
def _wrapped(*args, **kwargs): # TODO(dcramer): we want to tag a transaction ID, but overriding # the base on app.task seems to cause problems w/ Celery internals transaction_id = kwargs.pop("__transaction_id", None) key = "jobs.duration" if stat_suffix: instance = "{}.{}".format(name, stat_suffix(*args, **kwargs)) else: instance = name Raven.tags_context({"task_name": name, "transaction_id": transaction_id}) with metrics.timer(key, instance=instance), track_memory_usage("jobs.memory_change", instance=instance): try: result = func(*args, **kwargs) finally: Raven.context.clear() return result
def save_event(cache_key=None, data=None, start_time=None, event_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import tsdb if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] if data is None: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'}) return project = data.pop('project') delete_raw_event(project, event_id, allow_hint_clear=True) Raven.tags_context({ 'project': project, }) try: manager = EventManager(data) manager.save(project) except HashDiscarded as exc: # TODO(jess): remove this before it goes out to a wider audience info_logger.info( 'discarded.hash', extra={ 'project_id': project, 'description': exc.message, } ) tsdb.incr(tsdb.models.project_total_received_discarded, project, timestamp=start_time) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing( 'events.time-to-process', time() - start_time, instance=data['platform'])
def index_event_tags(organization_id, project_id, event_id, tags, group_id=None, **kwargs): from sentry import tagstore Raven.tags_context({ 'project': project_id, }) for key, value in tags: tagkey, _ = tagstore.get_or_create_tag_key(project_id, key) tagvalue, _ = tagstore.get_or_create_tag_value(project_id, key, value) tagstore.create_event_tag( project_id=project_id, group_id=group_id, event_id=event_id, key_id=tagkey.id, value_id=tagvalue.id, )
def _wrapped(*args, **kwargs): key = 'jobs.duration' if stat_suffix: instance = '{}.{}'.format(name, stat_suffix(*args, **kwargs)) else: instance = name Raven.tags_context({'task_name': name}) with metrics.timer(key, instance=instance), \ track_memory_usage('jobs.memory_change', instance=instance): try: return func(*args, **kwargs) except SoftTimeLimitExceeded as error: Raven.context.merge({ 'fingerprint': [type(error).__name__, instance], }) raise finally: Raven.context.clear()
def _wrapped(*args, **kwargs): # TODO(dcramer): we want to tag a transaction ID, but overriding # the base on app.task seems to cause problems w/ Celery internals transaction_id = kwargs.pop('__transaction_id', None) key = 'jobs.duration' if stat_suffix: instance = '{}.{}'.format(name, stat_suffix(*args, **kwargs)) else: instance = name Raven.tags_context({ 'task_name': name, 'transaction_id': transaction_id, }) with metrics.timer(key, instance=instance), \ track_memory_usage('jobs.memory_change', instance=instance): try: result = func(*args, **kwargs) finally: Raven.context.clear() return result
def _parse_header(self, request, project): try: auth_vars = extract_auth_vars(request) except (IndexError, ValueError): raise APIError("Invalid auth header") if not auth_vars: raise APIError("Client/server version mismatch: Unsupported client") server_version = auth_vars.get("sentry_version", "1.0") client = auth_vars.get("sentry_client", request.META.get("HTTP_USER_AGENT")) Raven.tags_context({"client": client}) Raven.tags_context({"protocol": server_version}) if server_version not in PROTOCOL_VERSIONS: raise APIError("Client/server version mismatch: Unsupported protocol version (%s)" % server_version) if not client: raise APIError("Client request error: Missing client version identifier") return auth_vars
def _wrapped(*args, **kwargs): # TODO(dcramer): we want to tag a transaction ID, but overriding # the base on app.task seems to cause problems w/ Celery internals transaction_id = kwargs.pop('__transaction_id', None) key = 'jobs.duration' if stat_suffix: instance = '{}.{}'.format(name, stat_suffix(*args, **kwargs)) else: instance = name Raven.tags_context({ 'task_name': name, 'transaction_id': transaction_id, }) with metrics.timer(key, instance=instance), \ track_memory_usage('jobs.memory_change', instance=instance), \ SqlQueryCountMonitor(name): try: result = func(*args, **kwargs) finally: Raven.context.clear() return result
def _parse_header(self, request, project): try: auth_vars = extract_auth_vars(request) except (IndexError, ValueError): raise APIError('Invalid auth header') if not auth_vars: raise APIError('Client/server version mismatch: Unsupported client') server_version = auth_vars.get('sentry_version', '1.0') client = auth_vars.get('sentry_client', request.META.get('HTTP_USER_AGENT')) Raven.tags_context({'client': client}) Raven.tags_context({'protocol': server_version}) if server_version not in PROTOCOL_VERSIONS: raise APIError('Client/server version mismatch: Unsupported protocol version (%s)' % server_version) if not client: raise APIError('Client request error: Missing client version identifier') return auth_vars
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): # TODO(ja): Refactor shared code with CspReportView. Especially, look at # the sentry_key override and test it. # A minidump submission as implemented by Breakpad and Crashpad or any # other library following the Mozilla Soccorro protocol is a POST request # without Origin or Referer headers. Therefore, we cannot validate the # origin of the request, but we *can* validate the "prod" key in future. if request.method != 'POST': return HttpResponseNotAllowed(['POST']) content_type = request.META.get('CONTENT_TYPE') # In case of multipart/form-data, the Content-Type header also includes # a boundary. Therefore, we cannot check for an exact match. if content_type is None or not content_type.startswith(self.content_types): raise APIError('Invalid Content-Type') request.user = AnonymousUser() project = self._get_project_from_id(project_id) helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) # This is yanking the auth from the querystring since it's not # in the POST body. This means we expect a `sentry_key` and # `sentry_version` to be set in querystring auth = helper.auth_from_request(request) key = helper.project_key_from_auth(auth) if key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) return super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, **kwargs )
def post_process_group(event, is_new, is_regression, is_sample, **kwargs): """ Fires post processing hooks for a group. """ from sentry.models import Project from sentry.rules.processor import RuleProcessor project_id = event.group.project_id Raven.tags_context({ 'project': project_id, }) project = Project.objects.get_from_cache(id=project_id) _capture_stats(event, is_new) rp = RuleProcessor(event, is_new, is_regression, is_sample) # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): safe_execute(callback, event, futures) for plugin in plugins.for_project(project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression, is_sample=is_sample, ) event_processed.send_robust( sender=post_process_group, project=project, group=event.group, event=event, )
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): raise APIForbidden('Invalid origin: %s' % (origin,)) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) project_ = helper.project_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise APIError('Unable to identify project') project = project_ elif project_ != project: raise APIError('Two different project were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) if auth.version != '2.0': if request.method == 'GET': # GET only requires an Origin/Referer check # If an Origin isn't passed, it's possible that the project allows no origin, # so we need to explicitly check for that here. If Origin is not None, # it can be safely assumed that it was checked previously and it's ok. if origin is None and not is_valid_origin(origin, project): # Special case an error message for a None origin when None wasn't allowed raise APIForbidden('Missing required Origin or Referer header') else: # Version 3 enforces secret key for server side requests if not auth.secret_key: raise APIForbidden('Missing required attribute in authentication header: sentry_secret') response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, **kwargs ) if origin: response['Access-Control-Allow-Origin'] = origin return response
def save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas, tsdb from sentry.models import ProjectKey if cache_key: data = default_cache.get(cache_key) if event_id is None and data is not None: event_id = data['event_id'] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop('project') delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr('events.failed', tags={'reason': 'cache', 'stage': 'post'}) return Raven.tags_context({ 'project': project_id, }) try: manager = EventManager(data) manager.save(project_id) except HashDiscarded: increment_list = [ (tsdb.models.project_total_received_discarded, project_id), ] try: project = Project.objects.get_from_cache(id=project_id) except Project.DoesNotExist: pass else: increment_list.extend([ (tsdb.models.project_total_blacklisted, project.id), (tsdb.models.organization_total_blacklisted, project.organization_id), ]) project_key = None if data.get('key_id') is not None: try: project_key = ProjectKey.objects.get_from_cache(id=data['key_id']) except ProjectKey.DoesNotExist: pass else: increment_list.append((tsdb.models.key_total_blacklisted, project_key.id)) quotas.refund( project, key=project_key, timestamp=start_time, ) tsdb.incr_multi( increment_list, timestamp=to_datetime(start_time) if start_time is not None else None, ) finally: if cache_key: default_cache.delete(cache_key) if start_time: metrics.timing( 'events.time-to-process', time() - start_time, instance=data['platform'])
def _dispatch(self, request, project_id=None, *args, **kwargs): request.user = AnonymousUser() try: project = self._get_project_from_id(project_id) except APIError as e: raise InvalidRequest(str(e)) if project: Raven.tags_context({'project': project.id}) origin = self.get_request_origin(request) if origin is not None: # This check is specific for clients who need CORS support if not project: raise InvalidRequest( 'Your client must be upgraded for CORS support.') if not is_valid_origin(origin, project): raise InvalidOrigin(origin) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: try: auth_vars = self._parse_header(request, project) except APIError as e: raise InvalidRequest(str(e)) try: project_, user = project_from_auth_vars(auth_vars) except APIError as error: return HttpResponse(six.text_type(error.msg), status=error.http_status) else: if user: request.user = user # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise InvalidRequest('Unable to identify project') project = project_ elif project_ != project: raise InvalidRequest('Project ID mismatch') else: Raven.tags_context({'project': project.id}) auth = Auth(auth_vars, is_public=bool(origin)) if auth.version >= 3: if request.method == 'GET': # GET only requires an Origin/Referer check # If an Origin isn't passed, it's possible that the project allows no origin, # so we need to explicitly check for that here. If Origin is not None, # it can be safely assumed that it was checked previously and it's ok. if origin is None and not is_valid_origin(origin, project): # Special case an error message for a None origin when None wasn't allowed raise InvalidRequest( 'Missing required Origin or Referer header') else: # Version 3 enforces secret key for server side requests if not auth.secret_key: raise InvalidRequest( 'Missing required attribute in authentication header: sentry_secret' ) try: response = super(APIView, self).dispatch(request, project=project, auth=auth, **kwargs) except APIError as error: response = HttpResponse(six.text_type(error.msg), content_type='text/plain', status=error.http_status) if isinstance( error, APIRateLimited) and error.retry_after is not None: response['Retry-After'] = str(error.retry_after) if origin: response['Access-Control-Allow-Origin'] = origin return response
def save_data(self, project, data, raw=False): # TODO: this function is way too damn long and needs refactored # the inner imports also suck so let's try to move it away from # the objects manager # TODO: culprit should default to "most recent" frame in stacktraces when # it's not provided. from sentry.plugins import plugins from sentry.models import Event, Project, EventMapping project = Project.objects.get_from_cache(id=project) Raven.tags_context({'project': project.id}) # First we pull out our top-level (non-data attr) kwargs event_id = data.pop('event_id') message = data.pop('message') culprit = data.pop('culprit') level = data.pop('level') time_spent = data.pop('time_spent') logger_name = data.pop('logger') server_name = data.pop('server_name') site = data.pop('site') date = data.pop('timestamp') checksum = data.pop('checksum') platform = data.pop('platform') if 'sentry.interfaces.Exception' in data: if 'values' not in data['sentry.interfaces.Exception']: data['sentry.interfaces.Exception'] = {'values': [data['sentry.interfaces.Exception']]} # convert stacktrace + exception into expanded exception if 'sentry.interfaces.Stacktrace' in data: data['sentry.interfaces.Exception']['values'][0]['stacktrace'] = data.pop('sentry.interfaces.Stacktrace') kwargs = { 'level': level, 'message': message, 'platform': platform, 'culprit': culprit or '', 'logger': logger_name, } event = Event( project=project, event_id=event_id, data=data, server_name=server_name, site=site, time_spent=time_spent, datetime=date, **kwargs ) # Calculate the checksum from the first highest scoring interface if not checksum: checksum = get_checksum_from_event(event) event.checksum = checksum group_kwargs = kwargs.copy() group_kwargs.update({ 'last_seen': date, 'first_seen': date, 'time_spent_total': time_spent or 0, 'time_spent_count': time_spent and 1 or 0, }) tags = data['tags'] tags.append(('level', LOG_LEVELS[level])) if logger: tags.append(('logger', logger_name)) if server_name: tags.append(('server_name', server_name)) if site: tags.append(('site', site)) for plugin in plugins.for_project(project): added_tags = safe_execute(plugin.get_tags, event) if added_tags: tags.extend(added_tags) try: group, is_new, is_sample = self._create_group( event=event, tags=data['tags'], **group_kwargs ) except Exception as exc: # TODO: should we mail admins when there are failures? try: logger.exception(u'Unable to process log entry: %s', exc) except Exception, exc: warnings.warn(u'Unable to process log entry: %s', exc) return
def post_process_group(event, is_new, is_regression, is_sample, is_new_group_environment, **kwargs): """ Fires post processing hooks for a group. """ # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor from sentry.tasks.servicehooks import process_service_hook # Re-bind Group since we're pickling the whole Event object # which may contain a stale Group. event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id project_id = event.group.project_id Raven.tags_context({ 'project': project_id, }) # Re-bind Project since we're pickling the whole Event object # which may contain a stale Project. event.project = Project.objects.get_from_cache(id=project_id) _capture_stats(event, is_new) # we process snoozes before rules as it might create a regression process_snoozes(event.group) rp = RuleProcessor(event, is_new, is_regression, is_new_group_environment) has_alert = False # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True safe_execute(callback, event, futures) if features.has( 'projects:servicehooks', project=event.project, ): allowed_events = set(['event.created']) if has_alert: allowed_events.add('event.alert') if allowed_events: for servicehook_id, events in _get_service_hooks(project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay( servicehook_id=servicehook_id, event=event, ) for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression, is_sample=is_sample, ) event_processed.send_robust( sender=post_process_group, project=event.project, group=event.group, event=event, primary_hash=kwargs.get('primary_hash'), )
project_, user = project_from_auth_vars(auth_vars) except APIError, error: return HttpResponse(unicode(error.msg), status=error.http_status) else: if user: request.user = user # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: return HttpResponse('Unable to identify project', content_type='text/plain', status=400) project = project_ elif project_ != project: return HttpResponse('Project ID mismatch', content_type='text/plain', status=400) else: Raven.tags_context({'project': project.id}) auth = Auth(auth_vars, is_public=bool(origin)) if auth.version >= 3: # Version 3 enforces secret key for server side requests if origin is None and not auth.secret_key: return HttpResponse('Missing required attribute in authentication header: sentry_secret', status=400) try: response = super(APIView, self).dispatch(request, project=project, auth=auth, **kwargs) except APIError, error: response = HttpResponse(unicode(error.msg), content_type='text/plain', status=error.http_status) if origin:
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): raise APIForbidden('Invalid origin: %s' % (origin, )) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) project_ = helper.project_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise APIError('Unable to identify project') project = project_ helper.context.bind_project(project) elif project_ != project: raise APIError('Two different project were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) if auth.version != '2.0': if not auth.secret_key: # If we're missing a secret_key, check if we are allowed # to do a CORS request. # If we're missing an Origin/Referrer header entirely, # we only want to support this on GET requests. By allowing # un-authenticated CORS checks for POST, we basially # are obsoleting our need for a secret key entirely. if origin is None and request.method != 'GET': raise APIForbidden( 'Missing required attribute in authentication header: sentry_secret' ) if not is_valid_origin(origin, project): raise APIForbidden( 'Missing required Origin or Referer header') response = super(APIView, self).dispatch(request=request, project=project, auth=auth, helper=helper, **kwargs) if origin: response['Access-Control-Allow-Origin'] = origin return response
def post_process_group(event, is_new, is_regression, is_sample, is_new_group_environment, **kwargs): """ Fires post processing hooks for a group. """ with redis.clusters.get('default').map() as client: result = client.set( u'pp:{}/{}'.format(event.project_id, event.event_id), u'{:.0f}'.format(time.time()), ex=60 * 60, nx=True, ) if not result.value: logger.info('post_process.skipped', extra={ 'project_id': event.project_id, 'event_id': event.event_id, 'reason': 'duplicate', }) return # NOTE: we must pass through the full Event object, and not an # event_id since the Event object may not actually have been stored # in the database due to sampling. from sentry.models import Project from sentry.models.group import get_group_with_redirect from sentry.rules.processor import RuleProcessor from sentry.tasks.servicehooks import process_service_hook # Re-bind Group since we're pickling the whole Event object # which may contain a stale Group. event.group, _ = get_group_with_redirect(event.group_id) event.group_id = event.group.id project_id = event.group.project_id Raven.tags_context({ 'project': project_id, }) # Re-bind Project since we're pickling the whole Event object # which may contain a stale Project. event.project = Project.objects.get_from_cache(id=project_id) _capture_stats(event, is_new) # we process snoozes before rules as it might create a regression process_snoozes(event.group) rp = RuleProcessor(event, is_new, is_regression, is_new_group_environment) has_alert = False # TODO(dcramer): ideally this would fanout, but serializing giant # objects back and forth isn't super efficient for callback, futures in rp.apply(): has_alert = True safe_execute(callback, event, futures) if features.has( 'projects:servicehooks', project=event.project, ): allowed_events = set(['event.created']) if has_alert: allowed_events.add('event.alert') if allowed_events: for servicehook_id, events in _get_service_hooks( project_id=event.project_id): if any(e in allowed_events for e in events): process_service_hook.delay( servicehook_id=servicehook_id, event=event, ) for plugin in plugins.for_project(event.project): plugin_post_process_group( plugin_slug=plugin.slug, event=event, is_new=is_new, is_regresion=is_regression, is_sample=is_sample, ) event_processed.send_robust( sender=post_process_group, project=event.project, group=event.group, event=event, primary_hash=kwargs.get('primary_hash'), )
def save_event(cache_key=None, data=None, start_time=None, event_id=None, project_id=None, **kwargs): """ Saves an event to the database. """ from sentry.event_manager import HashDiscarded, EventManager from sentry import quotas, tsdb from sentry.models import ProjectKey if cache_key: data = default_cache.get(cache_key) if data is not None: data = CanonicalKeyDict(data) if event_id is None and data is not None: event_id = data['event_id'] # only when we come from reprocessing we get a project_id sent into # the task. if project_id is None: project_id = data.pop('project') delete_raw_event(project_id, event_id, allow_hint_clear=True) # This covers two cases: where data is None because we did not manage # to fetch it from the default cache or the empty dictionary was # stored in the default cache. The former happens if the event # expired while being on the queue, the second happens on reprocessing # if the raw event was deleted concurrently while we held on to # it. This causes the node store to delete the data and we end up # fetching an empty dict. We could in theory not invoke `save_event` # in those cases but it's important that we always clean up the # reprocessing reports correctly or they will screw up the UI. So # to future proof this correctly we just handle this case here. if not data: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'post' }) return Raven.tags_context({ 'project': project_id, }) try: manager = EventManager(data) event = manager.save(project_id) # Always load attachments from the cache so we can later prune them. # Only save them if the event-attachments feature is active, though. if features.has('organizations:event-attachments', event.project.organization, actor=None): attachments = attachment_cache.get(cache_key) or [] for attachment in attachments: save_attachment(event, attachment) except HashDiscarded: increment_list = [ (tsdb.models.project_total_received_discarded, project_id), ] try: project = Project.objects.get_from_cache(id=project_id) except Project.DoesNotExist: pass else: increment_list.extend([ (tsdb.models.project_total_blacklisted, project.id), (tsdb.models.organization_total_blacklisted, project.organization_id), ]) project_key = None if data.get('key_id') is not None: try: project_key = ProjectKey.objects.get_from_cache( id=data['key_id']) except ProjectKey.DoesNotExist: pass else: increment_list.append( (tsdb.models.key_total_blacklisted, project_key.id)) quotas.refund( project, key=project_key, timestamp=start_time, ) tsdb.incr_multi( increment_list, timestamp=to_datetime(start_time) if start_time is not None else None, ) finally: if cache_key: default_cache.delete(cache_key) attachment_cache.delete(cache_key) if start_time: metrics.timing('events.time-to-process', time() - start_time, instance=data['platform'])
def _do_process_event(cache_key, start_time, event_id, process_task): from sentry.plugins import plugins data = default_cache.get(cache_key) if data is None: metrics.incr('events.failed', tags={ 'reason': 'cache', 'stage': 'process' }) error_logger.error('process.failed.empty', extra={'cache_key': cache_key}) return data = CanonicalKeyDict(data) project = data['project'] Raven.tags_context({ 'project': project, }) has_changed = False # Fetch the reprocessing revision reprocessing_rev = reprocessing.get_reprocessing_revision(project) # Stacktrace based event processors. These run before anything else. new_data = process_stacktraces(data) if new_data is not None: has_changed = True data = new_data # TODO(dcramer): ideally we would know if data changed by default # Default event processors. for plugin in plugins.all(version=2): processors = safe_execute(plugin.get_event_preprocessors, data=data, _with_transaction=False) for processor in (processors or ()): result = safe_execute(processor, data) if result: data = result has_changed = True assert data[ 'project'] == project, 'Project cannot be mutated by preprocessor' if has_changed: issues = data.get('processing_issues') try: if issues and create_failed_event( cache_key, project, list(issues.values()), event_id=event_id, start_time=start_time, reprocessing_rev=reprocessing_rev): return except RetryProcessing: # If `create_failed_event` indicates that we need to retry we # invoke outselves again. This happens when the reprocessing # revision changed while we were processing. process_task.delay(cache_key, start_time=start_time, event_id=event_id) return # We cannot persist canonical types in the cache, so we need to # downgrade this. if isinstance(data, CANONICAL_TYPES): data = dict(data.items()) default_cache.set(cache_key, data, 3600) save_event.delay(cache_key=cache_key, data=None, start_time=start_time, event_id=event_id, project_id=project)
def _dispatch(self, request, project_id=None, *args, **kwargs): request.user = AnonymousUser() try: project = self._get_project_from_id(project_id) except APIError as e: raise InvalidRequest(str(e)) if project: Raven.tags_context({'project': project.id}) origin = self.get_request_origin(request) if origin is not None: # This check is specific for clients who need CORS support if not project: raise InvalidRequest('Your client must be upgraded for CORS support.') if not is_valid_origin(origin, project): raise InvalidOrigin(origin) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: try: auth_vars = self._parse_header(request, project) except APIError as e: raise InvalidRequest(str(e)) try: project_, user = project_from_auth_vars(auth_vars) except APIError as error: return HttpResponse(six.text_type(error.msg), status=error.http_status) else: if user: request.user = user # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise InvalidRequest('Unable to identify project') project = project_ elif project_ != project: raise InvalidRequest('Project ID mismatch') else: Raven.tags_context({'project': project.id}) auth = Auth(auth_vars, is_public=bool(origin)) if auth.version >= 3: if request.method == 'GET': # GET only requires an Origin/Referer check # If an Origin isn't passed, it's possible that the project allows no origin, # so we need to explicitly check for that here. If Origin is not None, # it can be safely assumed that it was checked previously and it's ok. if origin is None and not is_valid_origin(origin, project): # Special case an error message for a None origin when None wasn't allowed raise InvalidRequest('Missing required Origin or Referer header') else: # Version 3 enforces secret key for server side requests if not auth.secret_key: raise InvalidRequest('Missing required attribute in authentication header: sentry_secret') try: response = super(APIView, self).dispatch(request, project=project, auth=auth, **kwargs) except APIError as error: response = HttpResponse(six.text_type(error.msg), content_type='text/plain', status=error.http_status) if isinstance(error, APIRateLimited) and error.retry_after is not None: response['Retry-After'] = str(error.retry_after) if origin: response['Access-Control-Allow-Origin'] = origin return response
def save_data(self, project, data, raw=False): # TODO: this function is way too damn long and needs refactored # the inner imports also suck so let's try to move it away from # the objects manager # TODO: culprit should default to "most recent" frame in stacktraces when # it's not provided. from sentry.plugins import plugins from sentry.models import Event, Project, EventMapping project = Project.objects.get_from_cache(id=project) Raven.tags_context({'project': project.id}) # First we pull out our top-level (non-data attr) kwargs event_id = data.pop('event_id') message = data.pop('message') culprit = data.pop('culprit') level = data.pop('level') time_spent = data.pop('time_spent') logger_name = data.pop('logger') server_name = data.pop('server_name') site = data.pop('site') date = data.pop('timestamp') checksum = data.pop('checksum') platform = data.pop('platform') if 'sentry.interfaces.Exception' in data: if 'values' not in data['sentry.interfaces.Exception']: data['sentry.interfaces.Exception'] = {'values': [data['sentry.interfaces.Exception']]} # convert stacktrace + exception into expanded exception if 'sentry.interfaces.Stacktrace' in data: data['sentry.interfaces.Exception']['values'][0]['stacktrace'] = data.pop('sentry.interfaces.Stacktrace') kwargs = { 'level': level, 'message': message, 'platform': platform, 'culprit': culprit or '', 'logger': logger_name, } event = Event( project=project, event_id=event_id, data=data, server_name=server_name, site=site, time_spent=time_spent, datetime=date, **kwargs ) # Calculate the checksum from the first highest scoring interface if not checksum: checksum = get_checksum_from_event(event) event.checksum = checksum group_kwargs = kwargs.copy() group_kwargs.update({ 'last_seen': date, 'first_seen': date, 'time_spent_total': time_spent or 0, 'time_spent_count': time_spent and 1 or 0, }) tags = data['tags'] tags.append(('level', LOG_LEVELS[level])) if logger: tags.append(('logger', logger_name)) if server_name: tags.append(('server_name', server_name)) if site: tags.append(('site', site)) for plugin in plugins.for_project(project): added_tags = safe_execute(plugin.get_tags, event) if added_tags: tags.extend(added_tags) try: group, is_new, is_sample = self._create_group( event=event, tags=data['tags'], **group_kwargs ) except Exception as exc: # TODO: should we mail admins when there are failures? try: logger.exception(u'Unable to process log entry: %s', exc) except Exception as exc: warnings.warn(u'Unable to process log entry: %s', exc) return using = group._state.db event.group = group # save the event unless its been sampled if not is_sample: sid = transaction.savepoint(using=using) try: event.save() except IntegrityError: transaction.savepoint_rollback(sid, using=using) return event transaction.savepoint_commit(sid, using=using) sid = transaction.savepoint(using=using) try: EventMapping.objects.create( project=project, group=group, event_id=event_id) except IntegrityError: transaction.savepoint_rollback(sid, using=using) return event transaction.savepoint_commit(sid, using=using) transaction.commit_unless_managed(using=using) if not raw: send_group_processors( group=group, event=event, is_new=is_new, is_sample=is_sample ) if getattr(settings, 'SENTRY_INDEX_SEARCH', settings.SENTRY_USE_SEARCH): index_event.delay(event) # TODO: move this to the queue if is_new and not raw: regression_signal.send_robust(sender=self.model, instance=group) return event