def test_querystring_as_pairlist(self): data = { "request": { "query_string": [ ["foo", "bar"], ["password", "hello"], ["the_secret", "hello"], ["a_password_here", "hello"], ["api_key", "secret_key"], ] } } proc = SensitiveDataFilter() proc.apply(data) assert "request" in data http = data["request"] assert http["query_string"] == [ ["foo", "bar"], ["password", FILTER_MASK], ["the_secret", FILTER_MASK], ["a_password_here", FILTER_MASK], ["api_key", FILTER_MASK], ]
def test_querystring_as_pairlist(self): data = { 'request': { 'query_string': [ ['foo', 'bar'], ['password', 'hello'], ['the_secret', 'hello'], ['a_password_here', 'hello'], ['api_key', 'secret_key'], ], } } proc = SensitiveDataFilter() proc.apply(data) assert 'request' in data http = data['request'] assert http['query_string'] == [ ['foo', 'bar'], ['password', FILTER_MASK], ['the_secret', FILTER_MASK], ['a_password_here', FILTER_MASK], ['api_key', FILTER_MASK], ]
def test_querystring_as_pairlist(self): data = { 'request': { 'query_string': [ ['foo', 'bar'], ['password', 'hello'], ['the_secret', 'hello'], ['a_password_here', 'hello'], ['api_key', 'secret_key'], ], } } proc = SensitiveDataFilter() proc.apply(data) assert 'request' in data http = data['request'] assert http['query_string'] == [ ['foo', 'bar'], ['password', FILTER_MASK], ['the_secret', FILTER_MASK], ['a_password_here', FILTER_MASK], ['api_key', FILTER_MASK], ]
def test_extra(self): data = {"extra": VARS} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("extra" in data) self._check_vars_sanitized(data["extra"], proc)
def test_extra(self): data = {'extra': VARS} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('extra' in data) self._check_vars_sanitized(data['extra'], proc)
def test_extra(self): data = {"extra": VARS} proc = SensitiveDataFilter() proc.apply(data) assert "extra" in data self._check_vars_sanitized(data["extra"], proc)
def test_extra(self): data = {'extra': VARS} proc = SensitiveDataFilter() proc.apply(data) assert 'extra' in data self._check_vars_sanitized(data['extra'], proc)
def test_querystring_as_string_with_partials(self): data = {"request": {"query_string": "foo=bar&password&baz=bar"}} proc = SensitiveDataFilter() proc.apply(data) assert "request" in data http = data["request"] assert http["query_string"] == "foo=bar&password&baz=bar"
def test_querystring_as_string_with_partials(self): data = {"request": {"query_string": "foo=bar&password&baz=bar"}} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("request" in data) http = data["request"] self.assertEquals(http["query_string"], "foo=bar&password&baz=bar" % dict(m=proc.MASK))
def test_user(self): data = {"user": {"username": "******", "data": VARS}} proc = SensitiveDataFilter() proc.apply(data) assert "user" in data assert data["user"]["username"] == "secret" self._check_vars_sanitized(data["user"]["data"], proc)
def test_does_not_fail_on_non_string(self): data = { 'extra': { 'foo': 1, }, } proc = SensitiveDataFilter() proc.apply(data) self.assertEquals(data['extra'], {'foo': 1})
def test_empty_field(self): data = { 'extra': { 'foobar': 'xxx', }, } proc = SensitiveDataFilter(fields=['']) proc.apply(data) assert data['extra'] == {'foobar': 'xxx'}
def test_does_sanitize_social_security_number(self): data = { 'extra': { 's': '123-45-6789', }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_does_sanitize_public_key(self): data = { 'extra': { 's': PUBLIC_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_explicit_fields(self): data = { 'extra': { 'mystuff': 'xxx', }, } proc = SensitiveDataFilter(fields=['mystuff']) proc.apply(data) assert data['extra']['mystuff'] == FILTER_MASK
def test_exclude_fields_on_field_value(self): data = { 'extra': { 'foobar': '123-45-6789', }, } proc = SensitiveDataFilter(exclude_fields=['foobar']) proc.apply(data) assert data['extra'] == {'foobar': '123-45-6789'}
def test_does_not_fail_on_non_string(self): data = { 'extra': { 'foo': 1, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'foo': 1}
def test_exclude_fields_on_field_name(self): data = { 'extra': { 'password': '******', }, } proc = SensitiveDataFilter(exclude_fields=['password']) proc.apply(data) assert data['extra'] == {'password': '******'}
def test_does_sanitize_social_security_number(self): data = { 'extra': { 's': '123-45-6789', }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_explicit_fields(self): data = { 'extra': { 'mystuff': 'xxx', }, } proc = SensitiveDataFilter(fields=['mystuff']) proc.apply(data) assert data['extra']['mystuff'] == FILTER_MASK
def test_does_sanitize_rsa_private_key(self): data = { 'extra': { 's': RSA_PRIVATE_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_exclude_fields_on_field_name(self): data = { 'extra': { 'password': '******', }, } proc = SensitiveDataFilter(exclude_fields=['password']) proc.apply(data) assert data['extra'] == {'password': '******'}
def test_does_sanitize_public_key(self): data = { 'extra': { 's': PUBLIC_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_empty_field(self): data = { 'extra': { 'foobar': 'xxx', }, } proc = SensitiveDataFilter(fields=['']) proc.apply(data) assert data['extra'] == {'foobar': 'xxx'}
def test_exclude_fields_on_field_value(self): data = { 'extra': { 'foobar': '123-45-6789', }, } proc = SensitiveDataFilter(exclude_fields=['foobar']) proc.apply(data) assert data['extra'] == {'foobar': '123-45-6789'}
def test_does_sanitize_rsa_private_key(self): data = { 'extra': { 's': RSA_PRIVATE_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def process(self, request, project, auth, data, **kwargs): event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) if rate_limit is not None and rate_limit.is_limited: raise APIRateLimited(rate_limit.retry_after) result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if content_encoding == 'gzip': data = decompress_gzip(data) elif content_encoding == 'deflate': data = decompress_deflate(data) elif not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e))) # mutates data manager = EventManager(data) data = manager.normalize() # insert IP address if not available if auth.is_public: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # We filter data immediately before it ever gets into the queue inst = SensitiveDataFilter() inst.apply(data) # mutates data (strips a lot of context if not queued) insert_data_to_database(data) logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id) return event_id
def test_http(self): data = {"request": {"data": VARS, "env": VARS, "headers": VARS, "cookies": VARS}} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("request" in data) http = data["request"] for n in ("data", "env", "headers", "cookies"): self.assertTrue(n in http) self._check_vars_sanitized(http[n], proc)
def test_sanitize_additional_sensitive_fields(self): additional_sensitive_dict = {'fieldy_field': 'value', 'moar_other_field': 'another value'} data = {'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))} proc = SensitiveDataFilter(additional_sensitive_dict.keys()) proc.apply(data) for field in additional_sensitive_dict.keys(): assert data['extra'][field] == FILTER_MASK self._check_vars_sanitized(data['extra'], proc)
def test_sanitize_additional_sensitive_fields(self): additional_sensitive_dict = {'fieldy_field': 'value', 'moar_other_field': 'another value'} data = {'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))} proc = SensitiveDataFilter(additional_sensitive_dict.keys()) proc.apply(data) for field in additional_sensitive_dict.keys(): assert data['extra'][field] == FILTER_MASK self._check_vars_sanitized(data['extra'], proc)
def test_contexts(self): data = {"contexts": {"secret": VARS, "biz": VARS}} proc = SensitiveDataFilter() proc.apply(data) assert "contexts" in data assert "secret" in data["contexts"] assert "biz" in data["contexts"] self._check_vars_sanitized(data["contexts"]["secret"], proc) self._check_vars_sanitized(data["contexts"]["biz"], proc)
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['data'], FILTER_MASK)
def test_sanitize_http_body(self): data = { "request": { "data": '{"email":"*****@*****.**","password":"******"}' } } proc = SensitiveDataFilter() proc.apply(data) assert "request" in data http = data["request"] assert http["data"] == FILTER_MASK
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['data'] == FILTER_MASK
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['data'] == FILTER_MASK
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['data'], FILTER_MASK)
def test_stacktrace(self): data = {"stacktrace": {"frames": [{"vars": VARS}]}} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("stacktrace" in data) stack = data["stacktrace"] self.assertTrue("frames" in stack) self.assertEquals(len(stack["frames"]), 1) frame = stack["frames"][0] self.assertTrue("vars" in frame) self._check_vars_sanitized(frame["vars"], proc)
def test_csp_blocked_uri(self): data = { 'sentry.interfaces.Csp': { 'blocked_uri': 'https://example.com/?foo=4571234567890111&bar=baz', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Csp' in data csp = data['sentry.interfaces.Csp'] assert csp['blocked_uri'] == 'https://example.com/?foo=[Filtered]&bar=baz'
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['query_string'] == 'foo=bar&password&baz=bar'
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['query_string'] == 'foo=bar&password&baz=bar'
def test_csp_blocked_uri(self): data = { 'csp': { 'blocked_uri': 'https://example.com/?foo=4571234567890111&bar=baz', } } proc = SensitiveDataFilter() proc.apply(data) assert 'csp' in data csp = data['csp'] assert csp['blocked_uri'] == 'https://example.com/?foo=[Filtered]&bar=baz'
def test_breadcrumb_message(self): data = { 'breadcrumbs': { 'values': [{ 'message': "SELECT session_key FROM django_session WHERE session_key = 'abcdefg'", }], }, } proc = SensitiveDataFilter(fields=['session_key']) proc.apply(data) assert data['breadcrumbs']['values'][0]['message'] == FILTER_MASK
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=FILTER_MASK))
def test_stacktrace(self): data = {"stacktrace": {"frames": [{"vars": VARS}]}} proc = SensitiveDataFilter() proc.apply(data) assert "stacktrace" in data stack = data["stacktrace"] assert "frames" in stack assert len(stack["frames"]) == 1 frame = stack["frames"][0] assert "vars" in frame self._check_vars_sanitized(frame["vars"], proc)
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=proc.MASK))
def process(self, request, project, auth, data, **kwargs): event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) if rate_limit is not None and rate_limit.is_limited: raise APIRateLimited(rate_limit.retry_after) result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if content_encoding == 'gzip': data = decompress_gzip(data) elif content_encoding == 'deflate': data = decompress_deflate(data) elif not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e))) # mutates data manager = EventManager(data) data = manager.normalize() # insert IP address if not available if auth.is_public: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # We filter data immediately before it ever gets into the queue inst = SensitiveDataFilter() inst.apply(data) # mutates data (strips a lot of context if not queued) insert_data_to_database(data) logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id) return event_id
def test_querystring_as_string_with_partials(self): data = { 'request': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('request' in data) http = data['request'] self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=proc.MASK))
def test_user(self): data = { 'sentry.interfaces.User': { 'username': '******', 'data': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.User' in data assert data['sentry.interfaces.User']['username'] == 'secret' self._check_vars_sanitized(data['sentry.interfaces.User']['data'], proc)
def test_breadcrumb_message(self): data = { "breadcrumbs": { "values": [{ "message": "SELECT session_key FROM django_session WHERE session_key = 'abcdefg'" }] } } proc = SensitiveDataFilter(fields=["session_key"]) proc.apply(data) assert data["breadcrumbs"]["values"][0]["message"] == FILTER_MASK
def test_user(self): data = { 'sentry.interfaces.User': { 'username': '******', 'data': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.User' in data assert data['sentry.interfaces.User']['username'] == 'secret' self._check_vars_sanitized(data['sentry.interfaces.User']['data'], proc)
def test_user_extra_data(self): """ Make sure we filter non-standard user entries, which all end up in user["data"] after normalization. See https://github.com/getsentry/semaphore/blob/5b6e46f6159843451027b5217902c81e134d7c40/general/src/protocol/user.rs#L54. """ data = {"user": {"data": VARS}} proc = SensitiveDataFilter() proc.apply(data) assert "user" in data self._check_vars_sanitized(data["user"]["data"], proc)
def test_csp_blocked_uri(self): data = { "csp": { "blocked_uri": "https://example.com/?foo=4571234567890111&bar=baz" } } proc = SensitiveDataFilter() proc.apply(data) assert "csp" in data csp = data["csp"] assert csp[ "blocked_uri"] == "https://example.com/?foo=[Filtered]&bar=baz"
def test_querystring_as_string(self): data = { "request": { "query_string": "foo=bar&password=hello&the_secret=hello" "&a_password_here=hello&api_key=secret_key" } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("request" in data) http = data["request"] self.assertEquals( http["query_string"], "foo=bar&password=%(m)s&the_secret=%(m)s" "&a_password_here=%(m)s&api_key=%(m)s" % dict(m=proc.MASK), )
def test_contexts(self): data = { 'contexts': { 'secret': VARS, 'biz': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'contexts' in data assert 'secret' in data['contexts'] assert 'biz' in data['contexts'] self._check_vars_sanitized(data['contexts']['secret'], proc) self._check_vars_sanitized(data['contexts']['biz'], proc)
def test_querystring_as_string(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password=hello&the_secret=hello' '&a_password_here=hello&api_key=secret_key', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals( http['query_string'], 'foo=bar&password=%(m)s&the_secret=%(m)s' '&a_password_here=%(m)s&api_key=%(m)s' % dict(m=proc.MASK))
def test_stacktrace(self): data = { 'sentry.interfaces.Stacktrace': { 'frames': [{'vars': VARS}], } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Stacktrace' in data) stack = data['sentry.interfaces.Stacktrace'] self.assertTrue('frames' in stack) self.assertEquals(len(stack['frames']), 1) frame = stack['frames'][0] self.assertTrue('vars' in frame) self._check_vars_sanitized(frame['vars'], proc)
def test_querystring_as_string(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password=hello&the_secret=hello' '&a_password_here=hello&api_key=secret_key', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['query_string'] == ( 'foo=bar&password=%(m)s&the_secret=%(m)s' '&a_password_here=%(m)s&api_key=%(m)s' % {'m': FILTER_MASK} )
def test_stacktrace(self): data = { 'sentry.interfaces.Stacktrace': { 'frames': [{'vars': VARS}], } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Stacktrace' in data stack = data['sentry.interfaces.Stacktrace'] assert 'frames' in stack assert len(stack['frames']) == 1 frame = stack['frames'][0] assert 'vars' in frame self._check_vars_sanitized(frame['vars'], proc)
def test_http(self): data = { 'sentry.interfaces.Http': { 'data': VARS, 'env': VARS, 'headers': VARS, 'cookies': VARS, } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] for n in ('data', 'env', 'headers', 'cookies'): self.assertTrue(n in http) self._check_vars_sanitized(http[n], proc)