def post(self, request, project, auth, helper, **kwargs): data = helper.safely_load_json_string(request.body) # Do origin check based on the `document-uri` key as explained # in `_dispatch`. try: report = data['csp-report'] except KeyError: raise APIError('Missing csp-report') origin = report.get('document-uri') # No idea, but this is garbage if origin == 'about:blank': raise APIForbidden('Invalid document-uri') if not is_valid_origin(origin, project): raise APIForbidden('Invalid document-uri') response_or_event_id = self.process(request, project=project, auth=auth, helper=helper, data=report, **kwargs) if isinstance(response_or_event_id, HttpResponse): return response_or_event_id return HttpResponse(status=201)
def post(self, request, project, helper, **kwargs): data = helper.safely_load_json_string(request.body) # Do origin check based on the `document-uri` key as explained # in `_dispatch`. try: report = data['csp-report'] except KeyError: raise APIError('Missing csp-report') origin = report.get('document-uri') # No idea, but this is garbage if origin == 'about:blank': raise APIForbidden('Invalid document-uri') if not is_valid_origin(origin, project): if project: tsdb.incr(tsdb.models.project_total_received_cors, project.id) raise APIForbidden('Invalid document-uri') # Attach on collected meta data. This data obviously isn't a part # of the spec, but we need to append to the report sentry specific things. report['_meta'] = { 'release': request.GET.get('sentry_release'), } response_or_event_id = self.process(request, project=project, helper=helper, data=report, **kwargs) if isinstance(response_or_event_id, HttpResponse): return response_or_event_id return HttpResponse(status=201)
def post(self, request, project, helper, key, **kwargs): json_body = safely_load_json_string(request.body) report_type = self.security_report_type(json_body) if report_type is None: raise APIError('Unrecognized security report type') interface = get_interface(report_type) try: instance = interface.from_raw(json_body) except jsonschema.ValidationError as e: raise APIError('Invalid security report: %s' % str(e).splitlines()[0]) # Do origin check based on the `document-uri` key as explained in `_dispatch`. origin = instance.get_origin() if not is_valid_origin(origin, project): if project: track_outcome(project.organization_id, project.id, key.id, 'invalid', FilterStatKeys.CORS) raise APIForbidden('Invalid origin') data = { 'interface': interface.path, 'report': instance, 'release': request.GET.get('sentry_release'), 'environment': request.GET.get('sentry_environment'), } self.process(request, project=project, helper=helper, data=data, key=key, **kwargs) return HttpResponse(content_type='application/javascript', status=201)
def _dispatch(self, request, helper, project_config, origin=None, *args, **kwargs): request.user = AnonymousUser() project = project_config.project config = project_config.config allowed = config.get('allowed_domains') if origin is not None: if not is_valid_origin(origin, allowed=allowed): track_outcome( project_config.organization_id, project_config.project_id, None, Outcome.INVALID, FilterStatKeys.CORS) raise APIForbidden('Invalid origin: %s' % (origin,)) auth = self._parse_header(request, project_config) key = helper.project_key_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if six.text_type(key.project_id) != six.text_type(project_config.project_id): raise APIError('Two different projects were specified') helper.context.bind_auth(auth) response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, project_config=project_config, **kwargs ) return response
def post(self, request, project, event_id, project_config, **kwargs): if not features.has("organizations:event-attachments", project.organization, actor=request.user): raise APIForbidden( "Event attachments are not enabled for this organization.") project_id = project_config.project_id if len(request.FILES) == 0: return HttpResponse(status=400) for name, uploaded_file in six.iteritems(request.FILES): file = File.objects.create( name=uploaded_file.name, type="event.attachment", headers={"Content-Type": uploaded_file.content_type}, ) file.putfile(uploaded_file) EventAttachment.objects.create(project_id=project_id, event_id=event_id, name=uploaded_file.name, file=file) return HttpResponse(status=201)
def post(self, request, project, helper, **kwargs): json_body = safely_load_json_string(request.body) report_type = self.security_report_type(json_body) if report_type is None: raise APIError('Unrecognized security report type') interface = get_interface(report_type) try: instance = interface.from_raw(json_body) except jsonschema.ValidationError as e: raise APIError('Invalid security report: %s' % str(e).splitlines()[0]) # Do origin check based on the `document-uri` key as explained in `_dispatch`. origin = instance.get_origin() if not is_valid_origin(origin, project): if project: tsdb.incr(tsdb.models.project_total_received_cors, project.id) raise APIForbidden('Invalid origin') data = { 'interface': interface.path, 'report': instance, 'release': request.GET.get('sentry_release'), 'environment': request.GET.get('sentry_environment'), } response_or_event_id = self.process(request, project=project, helper=helper, data=data, **kwargs) if isinstance(response_or_event_id, HttpResponse): return response_or_event_id return HttpResponse(content_type='application/javascript', status=201)
def process(self, request, project, key, auth, helper, data, attachments=None, **kwargs): metrics.incr('events.total', skip_internal=False) if not data: raise APIError('No JSON data was found') remote_addr = request.META['REMOTE_ADDR'] event_manager = EventManager( data, project=project, key=key, auth=auth, client_ip=remote_addr, user_agent=helper.context.agent, version=auth.version, content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''), ) del data self.pre_normalize(event_manager, helper) event_manager.normalize() data = event_manager.get_data() dict_data = dict(data) data_size = len(json.dumps(dict_data)) if data_size > 10000000: metrics.timing('events.size.rejected', data_size) raise APIForbidden("Event size exceeded 10MB after normalization.") return process_event(event_manager, project, key, remote_addr, helper, attachments)
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): tsdb.incr(tsdb.models.project_total_received_cors, project.id) raise APIForbidden('Invalid origin: %s' % (origin, )) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) key = helper.project_key_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: project = Project.objects.get_from_cache(id=key.project_id) helper.context.bind_project(project) elif key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) # Explicitly bind Organization so we don't implicitly query it later # this just allows us to comfortably assure that `project.organization` is safe. # This also allows us to pull the object from cache, instead of being # implicitly fetched from database. project.organization = Organization.objects.get_from_cache( id=project.organization_id) response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, key=key, **kwargs ) if origin: if origin == 'null': # If an Origin is `null`, but we got this far, that means # we've gotten past our CORS check for some reason. But the # problem is that we can't return "null" as a valid response # to `Access-Control-Allow-Origin` and we don't have another # value to work with, so just allow '*' since they've gotten # this far. response['Access-Control-Allow-Origin'] = '*' else: response['Access-Control-Allow-Origin'] = origin return response
def process(self, request, project, key, auth, helper, data, project_config, attachments=None, **kwargs): disable_transaction_events() metrics.incr("events.total", skip_internal=False) project_id = project_config.project_id organization_id = project_config.organization_id if not data: track_outcome(organization_id, project_id, key.id, Outcome.INVALID, "no_data") raise APIError("No JSON data was found") remote_addr = request.META["REMOTE_ADDR"] event_manager = EventManager( data, project=project, key=key, auth=auth, client_ip=remote_addr, user_agent=helper.context.agent, version=auth.version, content_encoding=request.META.get("HTTP_CONTENT_ENCODING", ""), project_config=project_config, ) del data self.pre_normalize(event_manager, helper) event_manager.normalize() data = event_manager.get_data() dict_data = dict(data) data_size = len(json.dumps(dict_data)) if data_size > 10000000: metrics.timing("events.size.rejected", data_size) track_outcome( organization_id, project_id, key.id, Outcome.INVALID, "too_large", event_id=dict_data.get("event_id"), ) raise APIForbidden("Event size exceeded 10MB after normalization.") metrics.timing("events.size.data.post_storeendpoint", data_size) return process_event(event_manager, project, key, remote_addr, helper, attachments, project_config)
def post(self, request, project, helper, key, project_config, **kwargs): json_body = safely_load_json_string(request.body) report_type = self.security_report_type(json_body) if report_type is None: track_outcome( project_config.organization_id, project_config.project_id, key.id, Outcome.INVALID, "security_report_type", ) raise APIError("Unrecognized security report type") interface = get_interface(report_type) try: instance = interface.from_raw(json_body) except jsonschema.ValidationError as e: track_outcome( project_config.organization_id, project_config.project_id, key.id, Outcome.INVALID, "security_report", ) raise APIError("Invalid security report: %s" % str(e).splitlines()[0]) # Do origin check based on the `document-uri` key as explained in `_dispatch`. origin = instance.get_origin() if not is_valid_origin(origin, project): track_outcome( project_config.organization_id, project_config.project_id, key.id, Outcome.INVALID, FilterStatKeys.CORS, ) raise APIForbidden("Invalid origin") data = { "interface": interface.path, "report": instance, "release": request.GET.get("sentry_release"), "environment": request.GET.get("sentry_environment"), } self.process( request, project=project, helper=helper, data=data, key=key, project_config=project_config, **kwargs ) return HttpResponse(content_type="application/javascript", status=201)
def process(self, request, project, auth, data, **kwargs): event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) if rate_limit is not None and rate_limit.is_limited: raise APIRateLimited(rate_limit.retry_after) result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if content_encoding == 'gzip': data = decompress_gzip(data) elif content_encoding == 'deflate': data = decompress_deflate(data) elif not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e))) # mutates data manager = EventManager(data) data = manager.normalize() # insert IP address if not available if auth.is_public: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # We filter data immediately before it ever gets into the queue inst = SensitiveDataFilter() inst.apply(data) # mutates data (strips a lot of context if not queued) insert_data_to_database(data) logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id) return event_id
def post(self, request, project, auth, helper, **kwargs): data = helper.safely_load_json_string(request.body) # Do origin check based on the `document-uri` key as explained # in `_dispatch`. try: report = data['csp-report'] except KeyError: raise APIError('Missing csp-report') origin = report.get('document-uri') # No idea, but this is garbage if origin == 'about:blank': raise APIForbidden('Invalid document-uri') if not is_valid_origin(origin, project): raise APIForbidden('Invalid document-uri') # An invalid CSP report must go against quota if not is_valid_csp_report(report, project): app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.project_total_blacklisted, project.id), (app.tsdb.models.organization_total_received, project.organization_id), (app.tsdb.models.organization_total_blacklisted, project.organization_id), ]) metrics.incr('events.blacklisted') raise APIForbidden('Rejected CSP report') response_or_event_id = self.process( request, project=project, auth=auth, helper=helper, data=report, **kwargs ) if isinstance(response_or_event_id, HttpResponse): return response_or_event_id return HttpResponse(status=201)
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): track_outcome(project.organization_id, project.id, None, Outcome.INVALID, FilterStatKeys.CORS) raise APIForbidden('Invalid origin: %s' % (origin, )) auth = self._parse_header(request, helper, project) key = helper.project_key_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: project = Project.objects.get_from_cache(id=key.project_id) helper.context.bind_project(project) elif key.project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) # Explicitly bind Organization so we don't implicitly query it later # this just allows us to comfortably assure that `project.organization` is safe. # This also allows us to pull the object from cache, instead of being # implicitly fetched from database. project.organization = Organization.objects.get_from_cache( id=project.organization_id) response = super(APIView, self).dispatch(request=request, project=project, auth=auth, helper=helper, key=key, **kwargs) return response
def process(self, request, project, auth, data, **kwargs): result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') if not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: validate_data(project, data, auth.client) except InvalidData, e: raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
def process_csp_report(self): """Only called from the CSP report endpoint.""" data = self._data try: interface = get_interface(data.pop('interface')) report = data.pop('report') except KeyError: raise APIForbidden('No report or interface data') # To support testing, we can either accept a built interface instance, or the raw data in # which case we build the instance ourselves try: instance = ( report if isinstance(report, interface) else interface.from_raw(report) ) except jsonschema.ValidationError as e: raise APIError('Invalid security report: %s' % str(e).splitlines()[0]) def clean(d): return dict(filter(lambda x: x[1], d.items())) data.update( { 'logger': 'csp', 'message': instance.get_message(), 'culprit': instance.get_culprit(), instance.path: instance.to_json(), 'tags': instance.get_tags(), 'errors': [], 'user': {'ip_address': self._client_ip}, # Construct a faux Http interface based on the little information we have # This is a bit weird, since we don't have nearly enough # information to create an Http interface, but # this automatically will pick up tags for the User-Agent # which is actually important here for CSP 'request': { 'url': instance.get_origin(), 'headers': clean( { 'User-Agent': self._user_agent, 'Referer': instance.get_referrer(), } ), }, } ) self._data = data
def process_csp_report(self): """Only called from the CSP report endpoint.""" data = self._data try: interface = get_interface(data.pop("interface")) report = data.pop("report") except KeyError: raise APIForbidden("No report or interface data") # To support testing, we can either accept a built interface instance, or the raw data in # which case we build the instance ourselves try: instance = report if isinstance( report, interface) else interface.from_raw(report) except jsonschema.ValidationError as e: raise APIError("Invalid security report: %s" % str(e).splitlines()[0]) def clean(d): return dict([x for x in d.items() if x[1]]) data.update({ "logger": "csp", "message": instance.get_message(), "culprit": instance.get_culprit(), instance.path: instance.to_json(), "tags": instance.get_tags(), "errors": [], "user": { "ip_address": self._client_ip }, # Construct a faux Http interface based on the little information we have # This is a bit weird, since we don't have nearly enough # information to create an Http interface, but # this automatically will pick up tags for the User-Agent # which is actually important here for CSP "request": { "url": instance.get_origin(), "headers": clean({ "User-Agent": self._user_agent, "Referer": instance.get_referrer() }), }, }) self._data = data
def post(self, request, project, event_id, relay_config, **kwargs): if not features.has('organizations:event-attachments', project.organization, actor=request.user): raise APIForbidden( "Event attachments are not enabled for this organization.") project_id = relay_config.project_id if len(request.FILES) == 0: return HttpResponse(status=400) for name, uploaded_file in six.iteritems(request.FILES): file = File.objects.create( name=uploaded_file.name, type='event.attachment', headers={'Content-Type': uploaded_file.content_type}, ) file.putfile(uploaded_file) # To avoid a race with EventManager which tries to set the group_id on attachments received before # the event, first insert the attachment, then lookup for the event for its group. event_attachment = EventAttachment.objects.create( project_id=project_id, event_id=event_id, name=uploaded_file.name, file=file, ) try: event = Event.objects.get( project_id=project_id, event_id=event_id, ) except Event.DoesNotExist: pass else: # If event was created but the group not defined, EventManager will take care of setting the # group to all dangling attachments if event.group_id is not None: EventAttachment.objects.filter( id=event_attachment.id, ).update( group_id=event.group_id, ) return HttpResponse(status=201)
def process(self, request, project, auth, data, **kwargs): event_received.send(ip=request.META['REMOTE_ADDR'], sender=type(self)) is_rate_limited = safe_execute(app.quotas.is_rate_limited, project=project) for plugin in plugins.all(): if safe_execute(plugin.is_rate_limited, project=project): is_rate_limited = True if is_rate_limited: raise APIRateLimited result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') if not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e))) # mutates data Group.objects.normalize_event_data(data) # insert IP address if not available if auth.is_public: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # mutates data (strips a lot of context if not queued) insert_data_to_database(data) logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id) return event_id
def handle_sentry(data, address): from sentry.coreapi import (project_from_auth_vars, decode_and_decompress_data, safely_load_json_string, validate_data, insert_data_to_database, APIError, APIForbidden) from sentry.models import Group from sentry.exceptions import InvalidData from sentry.plugins import plugins from sentry.utils.auth import parse_auth_header try: try: auth_header, data = data.split('\n\n', 1) except ValueError: raise APIError('missing auth header') try: auth_vars = parse_auth_header(auth_header) except (ValueError, IndexError): raise APIError('invalid auth header') project, user = project_from_auth_vars(auth_vars) result = plugins.first('has_perm', user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') client = auth_vars.get('sentry_client') if not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: validate_data(project, data, client) except InvalidData, e: raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e))) Group.objects.normalize_event_data(data) return insert_data_to_database(data)
def process(self, request, project, auth, data, **kwargs): if safe_execute(app.quotas.is_rate_limited, project=project): raise APIRateLimited for plugin in plugins.all(): if safe_execute(plugin.is_rate_limited, project=project): raise APIRateLimited result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') if not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData, e: raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
def process(self, request, project, auth, data, **kwargs): for plugin in plugins.all(): if safe_execute(plugin.is_rate_limited, project=project): return HttpResponse( 'Creation of this event was denied due to rate limiting.', content_type='text/plain', status=405) result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') if not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData, e: raise APIError(u'Invalid data: %s (%s)' % (unicode(e), type(e)))
def process(self, request, project, auth, helper, data, **kwargs): metrics.incr('events.total') if not data: raise APIError('No JSON data was found') remote_addr = request.META['REMOTE_ADDR'] data = LazyData( data=data, content_encoding=request.META.get('HTTP_CONTENT_ENCODING', ''), helper=helper, project=project, auth=auth, client_ip=remote_addr, ) event_received.send_robust( ip=remote_addr, project=project, sender=type(self), ) if helper.should_filter(project, data, ip_address=remote_addr): app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.project_total_blacklisted, project.id), (app.tsdb.models.organization_total_received, project.organization_id), (app.tsdb.models.organization_total_blacklisted, project.organization_id), ]) metrics.incr('events.blacklisted') event_filtered.send_robust( ip=remote_addr, project=project, sender=type(self), ) raise APIForbidden('Event dropped due to filter') # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project, _with_transaction=False) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) # XXX(dcramer): when the rate limiter fails we drop events to ensure # it cannot cascade if rate_limit is None or rate_limit.is_limited: if rate_limit is None: helper.log.debug( 'Dropped event due to error with rate limiter') app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.project_total_rejected, project.id), (app.tsdb.models.organization_total_received, project.organization_id), (app.tsdb.models.organization_total_rejected, project.organization_id), ]) metrics.incr( 'events.dropped', tags={ 'reason': rate_limit.reason_code if rate_limit else 'unknown', }) event_dropped.send_robust( ip=remote_addr, project=project, sender=type(self), reason_code=rate_limit.reason_code if rate_limit else None, ) if rate_limit is not None: raise APIRateLimited(rate_limit.retry_after) else: app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.organization_total_received, project.organization_id), ]) org_options = OrganizationOption.objects.get_all_values( project.organization_id) if org_options.get('sentry:require_scrub_ip_address', False): scrub_ip_address = True else: scrub_ip_address = project.get_option('sentry:scrub_ip_address', False) event_id = data['event_id'] # TODO(dcramer): ideally we'd only validate this if the event_id was # supplied by the user cache_key = 'ev:%s:%s' % ( project.id, event_id, ) if cache.get(cache_key) is not None: raise APIForbidden( 'An event with the same ID already exists (%s)' % (event_id, )) if org_options.get('sentry:require_scrub_data', False): scrub_data = True else: scrub_data = project.get_option('sentry:scrub_data', True) if scrub_data: # We filter data immediately before it ever gets into the queue sensitive_fields_key = 'sentry:sensitive_fields' sensitive_fields = (org_options.get(sensitive_fields_key, []) + project.get_option(sensitive_fields_key, [])) exclude_fields_key = 'sentry:safe_fields' exclude_fields = (org_options.get(exclude_fields_key, []) + project.get_option(exclude_fields_key, [])) if org_options.get('sentry:require_scrub_defaults', False): scrub_defaults = True else: scrub_defaults = project.get_option('sentry:scrub_defaults', True) inst = SensitiveDataFilter( fields=sensitive_fields, include_defaults=scrub_defaults, exclude_fields=exclude_fields, ) inst.apply(data) if scrub_ip_address: # We filter data immediately before it ever gets into the queue helper.ensure_does_not_have_ip(data) # mutates data (strips a lot of context if not queued) helper.insert_data_to_database(data) cache.set(cache_key, '', 60 * 5) helper.log.debug('New event received (%s)', event_id) event_accepted.send_robust( ip=remote_addr, data=data, project=project, sender=type(self), ) return event_id
def process_event(event_manager, project, key, remote_addr, helper, attachments, project_config): event_received.send_robust(ip=remote_addr, project=project, sender=process_event) start_time = time() data = event_manager.get_data() should_filter, filter_reason = event_manager.should_filter() del event_manager event_id = data["event_id"] if should_filter: signals_in_consumer = decide_signals_in_consumer() if not signals_in_consumer: # Mark that the event_filtered signal is sent. Do this before emitting # the outcome to avoid a potential race between OutcomesConsumer and # `event_filtered.send_robust` below. mark_signal_sent(project_config.project_id, event_id) track_outcome( project_config.organization_id, project_config.project_id, key.id, Outcome.FILTERED, filter_reason, event_id=event_id, ) metrics.incr("events.blacklisted", tags={"reason": filter_reason}, skip_internal=False) if not signals_in_consumer: event_filtered.send_robust(ip=remote_addr, project=project, sender=process_event) # relay will no longer be able to provide information about filter # status so to see the impact we're adding a way to turn on relay # like behavior here. if options.get("store.lie-about-filter-status"): return event_id raise APIForbidden("Event dropped due to filter: %s" % (filter_reason,)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute( quotas.is_rate_limited, project=project, key=key, _with_transaction=False ) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) # XXX(dcramer): when the rate limiter fails we drop events to ensure # it cannot cascade if rate_limit is None or rate_limit.is_limited: if rate_limit is None: api_logger.debug("Dropped event due to error with rate limiter") signals_in_consumer = decide_signals_in_consumer() if not signals_in_consumer: # Mark that the event_dropped signal is sent. Do this before emitting # the outcome to avoid a potential race between OutcomesConsumer and # `event_dropped.send_robust` below. mark_signal_sent(project_config.project_id, event_id) reason = rate_limit.reason_code if rate_limit else None track_outcome( project_config.organization_id, project_config.project_id, key.id, Outcome.RATE_LIMITED, reason, event_id=event_id, ) metrics.incr("events.dropped", tags={"reason": reason or "unknown"}, skip_internal=False) if not signals_in_consumer: event_dropped.send_robust( ip=remote_addr, project=project, reason_code=reason, sender=process_event ) if rate_limit is not None: raise APIRateLimited(rate_limit.retry_after) # TODO(dcramer): ideally we'd only validate this if the event_id was # supplied by the user cache_key = "ev:%s:%s" % (project_config.project_id, event_id) if cache.get(cache_key) is not None: track_outcome( project_config.organization_id, project_config.project_id, key.id, Outcome.INVALID, "duplicate", event_id=event_id, ) raise APIForbidden("An event with the same ID already exists (%s)" % (event_id,)) config = project_config.config datascrubbing_settings = config.get("datascrubbingSettings") or {} data = _scrub_event_data(data, datascrubbing_settings) # mutates data (strips a lot of context if not queued) helper.insert_data_to_database(data, start_time=start_time, attachments=attachments) cache.set(cache_key, "", 60 * 60) # Cache for 1 hour api_logger.debug("New event received (%s)", event_id) event_accepted.send_robust(ip=remote_addr, data=data, project=project, sender=process_event) return event_id
def apierror(message="Invalid data"): from sentry.coreapi import APIForbidden raise APIForbidden(message)
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): raise APIForbidden('Invalid origin: %s' % (origin,)) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) project_ = helper.project_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise APIError('Unable to identify project') project = project_ elif project_ != project: raise APIError('Two different project were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) if auth.version != '2.0': if request.method == 'GET': # GET only requires an Origin/Referer check # If an Origin isn't passed, it's possible that the project allows no origin, # so we need to explicitly check for that here. If Origin is not None, # it can be safely assumed that it was checked previously and it's ok. if origin is None and not is_valid_origin(origin, project): # Special case an error message for a None origin when None wasn't allowed raise APIForbidden('Missing required Origin or Referer header') else: # Version 3 enforces secret key for server side requests if not auth.secret_key: raise APIForbidden('Missing required attribute in authentication header: sentry_secret') response = super(APIView, self).dispatch( request=request, project=project, auth=auth, helper=helper, **kwargs ) if origin: response['Access-Control-Allow-Origin'] = origin return response
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): raise APIForbidden('Invalid origin: %s' % (origin, )) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) project_id = helper.project_id_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: project = Project.objects.get_from_cache(id=project_id) helper.context.bind_project(project) elif project_id != project.id: raise APIError('Two different projects were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) # Explicitly bind Organization so we don't implicitly query it later # this just allows us to comfortably assure that `project.organization` is safe. # This also allows us to pull the object from cache, instead of being # implicitly fetched from database. project.organization = Organization.objects.get_from_cache( id=project.organization_id) if auth.version != '2.0': if not auth.secret_key: # If we're missing a secret_key, check if we are allowed # to do a CORS request. # If we're missing an Origin/Referrer header entirely, # we only want to support this on GET requests. By allowing # un-authenticated CORS checks for POST, we basially # are obsoleting our need for a secret key entirely. if origin is None and request.method != 'GET': raise APIForbidden( 'Missing required attribute in authentication header: sentry_secret' ) if not is_valid_origin(origin, project): raise APIForbidden( 'Missing required Origin or Referer header') response = super(APIView, self).dispatch(request=request, project=project, auth=auth, helper=helper, **kwargs) if origin: if origin == 'null': # If an Origin is `null`, but we got this far, that means # we've gotten past our CORS check for some reason. But the # problem is that we can't return "null" as a valid response # to `Access-Control-Allow-Origin` and we don't have another # value to work with, so just allow '*' since they've gotten # this far. response['Access-Control-Allow-Origin'] = '*' else: response['Access-Control-Allow-Origin'] = origin return response
def process_event(event_manager, project, key, remote_addr, helper, attachments): event_received.send_robust(ip=remote_addr, project=project, sender=process_event) start_time = time() tsdb_start_time = to_datetime(start_time) should_filter, filter_reason = event_manager.should_filter() if should_filter: increment_list = [ (tsdb.models.project_total_received, project.id), (tsdb.models.project_total_blacklisted, project.id), (tsdb.models.organization_total_received, project.organization_id), (tsdb.models.organization_total_blacklisted, project.organization_id), (tsdb.models.key_total_received, key.id), (tsdb.models.key_total_blacklisted, key.id), ] try: increment_list.append( (FILTER_STAT_KEYS_TO_VALUES[filter_reason], project.id)) # should error when filter_reason does not match a key in FILTER_STAT_KEYS_TO_VALUES except KeyError: pass tsdb.incr_multi( increment_list, timestamp=tsdb_start_time, ) metrics.incr('events.blacklisted', tags={'reason': filter_reason}, skip_internal=False) event_filtered.send_robust( ip=remote_addr, project=project, sender=process_event, ) raise APIForbidden('Event dropped due to filter: %s' % (filter_reason, )) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(quotas.is_rate_limited, project=project, key=key, _with_transaction=False) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) # XXX(dcramer): when the rate limiter fails we drop events to ensure # it cannot cascade if rate_limit is None or rate_limit.is_limited: if rate_limit is None: api_logger.debug('Dropped event due to error with rate limiter') tsdb.incr_multi( [ (tsdb.models.project_total_received, project.id), (tsdb.models.project_total_rejected, project.id), (tsdb.models.organization_total_received, project.organization_id), (tsdb.models.organization_total_rejected, project.organization_id), (tsdb.models.key_total_received, key.id), (tsdb.models.key_total_rejected, key.id), ], timestamp=tsdb_start_time, ) metrics.incr( 'events.dropped', tags={ 'reason': rate_limit.reason_code if rate_limit else 'unknown', }, skip_internal=False, ) event_dropped.send_robust( ip=remote_addr, project=project, reason_code=rate_limit.reason_code if rate_limit else None, sender=process_event, ) if rate_limit is not None: raise APIRateLimited(rate_limit.retry_after) else: tsdb.incr_multi( [ (tsdb.models.project_total_received, project.id), (tsdb.models.organization_total_received, project.organization_id), (tsdb.models.key_total_received, key.id), ], timestamp=tsdb_start_time, ) org_options = OrganizationOption.objects.get_all_values( project.organization_id) data = event_manager.get_data() del event_manager event_id = data['event_id'] # TODO(dcramer): ideally we'd only validate this if the event_id was # supplied by the user cache_key = 'ev:%s:%s' % ( project.id, event_id, ) if cache.get(cache_key) is not None: raise APIForbidden('An event with the same ID already exists (%s)' % (event_id, )) scrub_ip_address = ( org_options.get('sentry:require_scrub_ip_address', False) or project.get_option('sentry:scrub_ip_address', False)) scrub_data = (org_options.get('sentry:require_scrub_data', False) or project.get_option('sentry:scrub_data', True)) if scrub_data: # We filter data immediately before it ever gets into the queue sensitive_fields_key = 'sentry:sensitive_fields' sensitive_fields = (org_options.get(sensitive_fields_key, []) + project.get_option(sensitive_fields_key, [])) exclude_fields_key = 'sentry:safe_fields' exclude_fields = (org_options.get(exclude_fields_key, []) + project.get_option(exclude_fields_key, [])) scrub_defaults = (org_options.get('sentry:require_scrub_defaults', False) or project.get_option('sentry:scrub_defaults', True)) SensitiveDataFilter( fields=sensitive_fields, include_defaults=scrub_defaults, exclude_fields=exclude_fields, ).apply(data) if scrub_ip_address: # We filter data immediately before it ever gets into the queue helper.ensure_does_not_have_ip(data) # mutates data (strips a lot of context if not queued) helper.insert_data_to_database(data, start_time=start_time, attachments=attachments) cache.set(cache_key, '', 60 * 5) api_logger.debug('New event received (%s)', event_id) event_accepted.send_robust( ip=remote_addr, data=data, project=project, sender=process_event, ) return event_id
def process(self, request, project, auth, data, **kwargs): event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project, _with_transaction=False) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) if rate_limit is not None and rate_limit.is_limited: app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.project_total_rejected, project.id), (app.tsdb.models.organization_total_received, project.organization_id), (app.tsdb.models.organization_total_rejected, project.organization_id), ]) raise APIRateLimited(rate_limit.retry_after) else: app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.organization_total_received, project.organization_id), ]) result = plugins.first('has_perm', request.user, 'create_event', project, version=1) if result is False: raise APIForbidden('Creation of this event was blocked') content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if content_encoding == 'gzip': data = decompress_gzip(data) elif content_encoding == 'deflate': data = decompress_deflate(data) elif not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e))) # mutates data manager = EventManager(data, version=auth.version) data = manager.normalize() scrub_ip_address = project.get_option('sentry:scrub_ip_address', False) # insert IP address if not available if auth.is_public and not scrub_ip_address: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # TODO(dcramer): ideally we'd only validate this if the event_id was # supplied by the user cache_key = 'ev:%s:%s' % ( project.id, event_id, ) if cache.get(cache_key) is not None: logger.warning( 'Discarded recent duplicate event from project %s/%s (id=%s)', project.organization.slug, project.slug, event_id) raise InvalidRequest('An event with the same ID already exists.') if project.get_option('sentry:scrub_data', True): # We filter data immediately before it ever gets into the queue inst = SensitiveDataFilter() inst.apply(data) if scrub_ip_address: # We filter data immediately before it ever gets into the queue ensure_does_not_have_ip(data) # mutates data (strips a lot of context if not queued) insert_data_to_database(data) cache.set(cache_key, '', 60 * 5) logger.debug('New event from project %s/%s (id=%s)', project.organization.slug, project.slug, event_id) return event_id
def _dispatch(self, request, helper, project_id=None, origin=None, *args, **kwargs): request.user = AnonymousUser() project = self._get_project_from_id(project_id) if project: helper.context.bind_project(project) Raven.tags_context(helper.context.get_tags_context()) if origin is not None: # This check is specific for clients who need CORS support if not project: raise APIError('Client must be upgraded for CORS support') if not is_valid_origin(origin, project): raise APIForbidden('Invalid origin: %s' % (origin, )) # XXX: It seems that the OPTIONS call does not always include custom headers if request.method == 'OPTIONS': response = self.options(request, project) else: auth = self._parse_header(request, helper, project) project_ = helper.project_from_auth(auth) # Legacy API was /api/store/ and the project ID was only available elsewhere if not project: if not project_: raise APIError('Unable to identify project') project = project_ helper.context.bind_project(project) elif project_ != project: raise APIError('Two different project were specified') helper.context.bind_auth(auth) Raven.tags_context(helper.context.get_tags_context()) if auth.version != '2.0': if not auth.secret_key: # If we're missing a secret_key, check if we are allowed # to do a CORS request. # If we're missing an Origin/Referrer header entirely, # we only want to support this on GET requests. By allowing # un-authenticated CORS checks for POST, we basially # are obsoleting our need for a secret key entirely. if origin is None and request.method != 'GET': raise APIForbidden( 'Missing required attribute in authentication header: sentry_secret' ) if not is_valid_origin(origin, project): raise APIForbidden( 'Missing required Origin or Referer header') response = super(APIView, self).dispatch(request=request, project=project, auth=auth, helper=helper, **kwargs) if origin: response['Access-Control-Allow-Origin'] = origin return response
def process(self, request, project, auth, helper, data, **kwargs): metrics.incr('events.total') remote_addr = request.META['REMOTE_ADDR'] event_received.send_robust(ip=remote_addr, sender=type(self)) if not is_valid_ip(remote_addr, project): app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.project_total_blacklisted, project.id), (app.tsdb.models.organization_total_received, project.organization_id), (app.tsdb.models.organization_total_blacklisted, project.organization_id), ]) metrics.incr('events.blacklisted') raise APIForbidden('Blacklisted IP address: %s' % (remote_addr, )) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project, _with_transaction=False) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) # XXX(dcramer): when the rate limiter fails we drop events to ensure # it cannot cascade if rate_limit is None or rate_limit.is_limited: if rate_limit is None: helper.log.debug( 'Dropped event due to error with rate limiter') app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.project_total_rejected, project.id), (app.tsdb.models.organization_total_received, project.organization_id), (app.tsdb.models.organization_total_rejected, project.organization_id), ]) metrics.incr('events.dropped') if rate_limit is not None: raise APIRateLimited(rate_limit.retry_after) else: app.tsdb.incr_multi([ (app.tsdb.models.project_total_received, project.id), (app.tsdb.models.organization_total_received, project.organization_id), ]) content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if isinstance(data, basestring): if content_encoding == 'gzip': data = helper.decompress_gzip(data) elif content_encoding == 'deflate': data = helper.decompress_deflate(data) elif not data.startswith('{'): data = helper.decode_and_decompress_data(data) data = helper.safely_load_json_string(data) # mutates data data = helper.validate_data(project, data) # mutates data manager = EventManager(data, version=auth.version) data = manager.normalize() org_options = OrganizationOption.objects.get_all_values( project.organization_id) if org_options.get('sentry:require_scrub_ip_address', False): scrub_ip_address = True else: scrub_ip_address = project.get_option('sentry:scrub_ip_address', False) # insert IP address if not available if auth.is_public and not scrub_ip_address: helper.ensure_has_ip(data, remote_addr) event_id = data['event_id'] # TODO(dcramer): ideally we'd only validate this if the event_id was # supplied by the user cache_key = 'ev:%s:%s' % ( project.id, event_id, ) if cache.get(cache_key) is not None: raise APIForbidden( 'An event with the same ID already exists (%s)' % (event_id, )) if org_options.get('sentry:require_scrub_data', False): scrub_data = True else: scrub_data = project.get_option('sentry:scrub_data', True) if scrub_data: # We filter data immediately before it ever gets into the queue sensitive_fields_key = 'sentry:sensitive_fields' sensitive_fields = (org_options.get(sensitive_fields_key, []) + project.get_option(sensitive_fields_key, [])) if org_options.get('sentry:require_scrub_defaults', False): scrub_defaults = True else: scrub_defaults = project.get_option('sentry:scrub_defaults', True) inst = SensitiveDataFilter( fields=sensitive_fields, include_defaults=scrub_defaults, ) inst.apply(data) if scrub_ip_address: # We filter data immediately before it ever gets into the queue helper.ensure_does_not_have_ip(data) # mutates data (strips a lot of context if not queued) helper.insert_data_to_database(data) cache.set(cache_key, '', 60 * 5) helper.log.debug('New event received (%s)', event_id) return event_id