def test_querystring_as_pairlist(self): data = { 'request': { 'query_string': [ ['foo', 'bar'], ['password', 'hello'], ['the_secret', 'hello'], ['a_password_here', 'hello'], ['api_key', 'secret_key'], ], } } proc = SensitiveDataFilter() proc.apply(data) assert 'request' in data http = data['request'] assert http['query_string'] == [ ['foo', 'bar'], ['password', FILTER_MASK], ['the_secret', FILTER_MASK], ['a_password_here', FILTER_MASK], ['api_key', FILTER_MASK], ]
def test_extra(self): data = {"extra": VARS} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("extra" in data) self._check_vars_sanitized(data["extra"], proc)
def test_sanitize_credit_card_within_value(self): proc = SensitiveDataFilter() result = proc.sanitize('foo', "'4571234567890111'") assert result == FILTER_MASK proc = SensitiveDataFilter() result = proc.sanitize('foo', "foo 4571234567890111") assert result == FILTER_MASK
def test_sanitize_credit_card_within_value(self): proc = SensitiveDataFilter() result = proc.sanitize('foo', "'4242424242424242'") self.assertEquals(result, proc.MASK) proc = SensitiveDataFilter() result = proc.sanitize('foo', "foo 4242424242424242") self.assertEquals(result, proc.MASK)
def test_extra(self): data = {'extra': VARS} proc = SensitiveDataFilter() proc.apply(data) assert 'extra' in data self._check_vars_sanitized(data['extra'], proc)
def test_querystring_as_string_with_partials(self): data = {"request": {"query_string": "foo=bar&password&baz=bar"}} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("request" in data) http = data["request"] self.assertEquals(http["query_string"], "foo=bar&password&baz=bar" % dict(m=proc.MASK))
def test_sanitize_url(self): proc = SensitiveDataFilter() result = proc.sanitize('foo', 'pg://*****:*****@localhost/1') self.assertEquals(result, 'pg://*****:*****@localhost/1' % proc.MASK) # Make sure we don't mess up any other url. # This url specifically if passed through urlunsplit(urlsplit()), # it'll change the value. result = proc.sanitize('foo', 'postgres:///path') self.assertEquals(result, 'postgres:///path')
def test_does_sanitize_rsa_private_key(self): data = { 'extra': { 's': RSA_PRIVATE_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_does_sanitize_social_security_number(self): data = { 'extra': { 's': '123-45-6789', }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_exclude_fields_on_field_name(self): data = { 'extra': { 'password': '******', }, } proc = SensitiveDataFilter(exclude_fields=['password']) proc.apply(data) assert data['extra'] == {'password': '******'}
def test_explicit_fields(self): data = { 'extra': { 'mystuff': 'xxx', }, } proc = SensitiveDataFilter(fields=['mystuff']) proc.apply(data) assert data['extra']['mystuff'] == FILTER_MASK
def test_does_not_fail_on_non_string(self): data = { 'extra': { 'foo': 1, }, } proc = SensitiveDataFilter() result = proc.apply(data) self.assertEquals(data['extra'], {'foo': 1})
def test_explicit_fields(self): data = { 'extra': { 'mystuff': 'xxx', }, } proc = SensitiveDataFilter(fields=['mystuff']) proc.apply(data) assert data['extra']['mystuff'] == FILTER_MASK
def test_exclude_fields_on_field_value(self): data = { 'extra': { 'foobar': '123-45-6789', }, } proc = SensitiveDataFilter(exclude_fields=['foobar']) proc.apply(data) assert data['extra'] == {'foobar': '123-45-6789'}
def test_empty_field(self): data = { 'extra': { 'foobar': 'xxx', }, } proc = SensitiveDataFilter(fields=['']) proc.apply(data) assert data['extra'] == {'foobar': 'xxx'}
def test_does_sanitize_private_key(self): data = { 'extra': { 's': PRIVATE_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_does_not_fail_on_non_string(self): data = { 'extra': { 'foo': 1, }, } proc = SensitiveDataFilter() result = proc.apply(data) self.assertEquals(data['extra'], {'foo': 1})
def test_does_not_fail_on_non_string(self): data = { 'extra': { 'foo': 1, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'foo': 1}
def test_does_sanitize_public_key(self): data = { 'extra': { 's': PUBLIC_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_empty_field(self): data = { 'extra': { 'foobar': 'xxx', }, } proc = SensitiveDataFilter(fields=['']) proc.apply(data) assert data['extra'] == {'foobar': 'xxx'}
def test_exclude_fields_on_field_name(self): data = { 'extra': { 'password': '******', }, } proc = SensitiveDataFilter(exclude_fields=['password']) proc.apply(data) assert data['extra'] == {'password': '******'}
def test_exclude_fields_on_field_value(self): data = { 'extra': { 'foobar': '123-45-6789', }, } proc = SensitiveDataFilter(exclude_fields=['foobar']) proc.apply(data) assert data['extra'] == {'foobar': '123-45-6789'}
def test_extra(self): data = { 'extra': VARS } proc = SensitiveDataFilter() proc.apply(data) assert 'extra' in data self._check_vars_sanitized(data['extra'], proc)
def test_does_sanitize_social_security_number(self): data = { 'extra': { 's': '123-45-6789', }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_does_not_fail_on_non_string(self): data = { 'extra': { 'foo': 1, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'foo': 1}
def test_does_sanitize_public_key(self): data = { 'extra': { 's': PUBLIC_KEY, }, } proc = SensitiveDataFilter() proc.apply(data) assert data['extra'] == {'s': FILTER_MASK}
def test_sanitize_url(self): proc = SensitiveDataFilter() result = proc.sanitize('foo', 'pg://*****:*****@localhost/1') assert result == 'pg://*****:*****@localhost/1' % FILTER_MASK result = proc.sanitize('foo', "foo 'redis://*****:*****@localhost:6379/0' bar") assert result == "foo 'redis://*****:*****@localhost:6379/0' bar" % FILTER_MASK result = proc.sanitize('foo', "'redis://*****:*****@localhost:6379/0'") assert result == "'redis://*****:*****@localhost:6379/0'" % FILTER_MASK result = proc.sanitize('foo', "foo redis://redis:foo@localhost:6379/0 bar") assert result == "foo redis://redis:%s@localhost:6379/0 bar" % FILTER_MASK result = proc.sanitize( 'foo', "foo redis://redis:foo@localhost:6379/0 bar pg://matt:foo@localhost/1" ) assert result == "foo redis://redis:%s@localhost:6379/0 bar pg://matt:%s@localhost/1" % ( FILTER_MASK, FILTER_MASK) # Make sure we don't mess up any other url. # This url specifically if passed through urlunsplit(urlsplit()), # it'll change the value. result = proc.sanitize('foo', 'postgres:///path') assert result == 'postgres:///path' # Don't be too overly eager within JSON strings an catch the right field. result = proc.sanitize( 'foo', '{"a":"https://localhost","b":"foo@localhost","c":"pg://*****:*****@localhost/1","d":"lol"}' ) assert result == '{"a":"https://localhost","b":"foo@localhost","c":"pg://*****:*****@localhost/1","d":"lol"}' % FILTER_MASK
def process(self, request, project, auth, data, **kwargs): event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) if rate_limit is not None and rate_limit.is_limited: raise APIRateLimited(rate_limit.retry_after) result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if content_encoding == 'gzip': data = decompress_gzip(data) elif content_encoding == 'deflate': data = decompress_deflate(data) elif not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e))) # mutates data manager = EventManager(data) data = manager.normalize() # insert IP address if not available if auth.is_public: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # We filter data immediately before it ever gets into the queue inst = SensitiveDataFilter() inst.apply(data) # mutates data (strips a lot of context if not queued) insert_data_to_database(data) logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id) return event_id
def test_http(self): data = {"request": {"data": VARS, "env": VARS, "headers": VARS, "cookies": VARS}} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("request" in data) http = data["request"] for n in ("data", "env", "headers", "cookies"): self.assertTrue(n in http) self._check_vars_sanitized(http[n], proc)
def test_sanitize_additional_sensitive_fields(self): additional_sensitive_dict = {'fieldy_field': 'value', 'moar_other_field': 'another value'} data = {'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))} proc = SensitiveDataFilter(additional_sensitive_dict.keys()) proc.apply(data) for field in additional_sensitive_dict.keys(): assert data['extra'][field] == FILTER_MASK self._check_vars_sanitized(data['extra'], proc)
def test_sanitize_additional_sensitive_fields(self): additional_sensitive_dict = {'fieldy_field': 'value', 'moar_other_field': 'another value'} data = {'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))} proc = SensitiveDataFilter(additional_sensitive_dict.keys()) proc.apply(data) for field in additional_sensitive_dict.keys(): assert data['extra'][field] == FILTER_MASK self._check_vars_sanitized(data['extra'], proc)
def test_contexts(self): data = {"contexts": {"secret": VARS, "biz": VARS}} proc = SensitiveDataFilter() proc.apply(data) assert "contexts" in data assert "secret" in data["contexts"] assert "biz" in data["contexts"] self._check_vars_sanitized(data["contexts"]["secret"], proc) self._check_vars_sanitized(data["contexts"]["biz"], proc)
def test_sanitize_http_body(self): data = { "request": { "data": '{"email":"*****@*****.**","password":"******"}' } } proc = SensitiveDataFilter() proc.apply(data) assert "request" in data http = data["request"] assert http["data"] == FILTER_MASK
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['data'] == FILTER_MASK
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() result = proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['data'], proc.MASK)
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['data'] == FILTER_MASK
def test_sanitize_http_body(self): data = { 'sentry.interfaces.Http': { 'data': '{"email":"*****@*****.**","password":"******"}', }, } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['data'], FILTER_MASK)
def test_csp_blocked_uri(self): data = { 'csp': { 'blocked_uri': 'https://example.com/?foo=4571234567890111&bar=baz', } } proc = SensitiveDataFilter() proc.apply(data) assert 'csp' in data csp = data['csp'] assert csp['blocked_uri'] == 'https://example.com/?foo=[Filtered]&bar=baz'
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['query_string'] == 'foo=bar&password&baz=bar'
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Http' in data http = data['sentry.interfaces.Http'] assert http['query_string'] == 'foo=bar&password&baz=bar'
def test_breadcrumb_message(self): data = { 'breadcrumbs': { 'values': [{ 'message': "SELECT session_key FROM django_session WHERE session_key = 'abcdefg'", }], }, } proc = SensitiveDataFilter(fields=['session_key']) proc.apply(data) assert data['breadcrumbs']['values'][0]['message'] == FILTER_MASK
def test_csp_blocked_uri(self): data = { 'sentry.interfaces.Csp': { 'blocked_uri': 'https://example.com/?foo=4571234567890111&bar=baz', } } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.Csp' in data csp = data['sentry.interfaces.Csp'] assert csp['blocked_uri'] == 'https://example.com/?foo=[Filtered]&bar=baz'
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=FILTER_MASK))
def test_querystring_as_string_with_partials(self): data = { 'sentry.interfaces.Http': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('sentry.interfaces.Http' in data) http = data['sentry.interfaces.Http'] self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=proc.MASK))
def test_stacktrace(self): data = {"stacktrace": {"frames": [{"vars": VARS}]}} proc = SensitiveDataFilter() proc.apply(data) assert "stacktrace" in data stack = data["stacktrace"] assert "frames" in stack assert len(stack["frames"]) == 1 frame = stack["frames"][0] assert "vars" in frame self._check_vars_sanitized(frame["vars"], proc)
def test_stacktrace(self): data = {"stacktrace": {"frames": [{"vars": VARS}]}} proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("stacktrace" in data) stack = data["stacktrace"] self.assertTrue("frames" in stack) self.assertEquals(len(stack["frames"]), 1) frame = stack["frames"][0] self.assertTrue("vars" in frame) self._check_vars_sanitized(frame["vars"], proc)
def process(self, request, project, auth, data, **kwargs): event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self)) # TODO: improve this API (e.g. make RateLimit act on __ne__) rate_limit = safe_execute(app.quotas.is_rate_limited, project=project) if isinstance(rate_limit, bool): rate_limit = RateLimit(is_limited=rate_limit, retry_after=None) if rate_limit is not None and rate_limit.is_limited: raise APIRateLimited(rate_limit.retry_after) result = plugins.first('has_perm', request.user, 'create_event', project) if result is False: raise APIForbidden('Creation of this event was blocked') content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '') if content_encoding == 'gzip': data = decompress_gzip(data) elif content_encoding == 'deflate': data = decompress_deflate(data) elif not data.startswith('{'): data = decode_and_decompress_data(data) data = safely_load_json_string(data) try: # mutates data validate_data(project, data, auth.client) except InvalidData as e: raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e))) # mutates data manager = EventManager(data) data = manager.normalize() # insert IP address if not available if auth.is_public: ensure_has_ip(data, request.META['REMOTE_ADDR']) event_id = data['event_id'] # We filter data immediately before it ever gets into the queue inst = SensitiveDataFilter() inst.apply(data) # mutates data (strips a lot of context if not queued) insert_data_to_database(data) logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id) return event_id
def test_breadcrumb_message(self): data = { "breadcrumbs": { "values": [{ "message": "SELECT session_key FROM django_session WHERE session_key = 'abcdefg'" }] } } proc = SensitiveDataFilter(fields=["session_key"]) proc.apply(data) assert data["breadcrumbs"]["values"][0]["message"] == FILTER_MASK
def test_querystring_as_string_with_partials(self): data = { 'request': { 'query_string': 'foo=bar&password&baz=bar', } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue('request' in data) http = data['request'] self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=proc.MASK))
def test_sanitize_url(self): proc = SensitiveDataFilter() result = proc.sanitize('foo', 'pg://*****:*****@localhost/1') self.assertEquals(result, 'pg://*****:*****@localhost/1' % FILTER_MASK) # Make sure we don't mess up any other url. # This url specifically if passed through urlunsplit(urlsplit()), # it'll change the value. result = proc.sanitize('foo', 'postgres:///path') self.assertEquals(result, 'postgres:///path') result = proc.sanitize('foo', "foo 'redis://*****:*****@localhost:6379/0' bar") self.assertEquals( result, "foo 'redis://*****:*****@localhost:6379/0' bar" % FILTER_MASK) result = proc.sanitize('foo', "'redis://*****:*****@localhost:6379/0'") self.assertEquals(result, "'redis://*****:*****@localhost:6379/0'" % FILTER_MASK) result = proc.sanitize('foo', "foo redis://redis:foo@localhost:6379/0 bar") self.assertEquals( result, "foo redis://redis:%s@localhost:6379/0 bar" % FILTER_MASK) result = proc.sanitize( 'foo', "foo redis://redis:foo@localhost:6379/0 bar pg://matt:foo@localhost/1" ) self.assertEquals( result, "foo redis://redis:%s@localhost:6379/0 bar pg://matt:%s@localhost/1" % (FILTER_MASK, FILTER_MASK))
def test_user(self): data = { 'sentry.interfaces.User': { 'username': '******', 'data': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.User' in data assert data['sentry.interfaces.User']['username'] == 'secret' self._check_vars_sanitized(data['sentry.interfaces.User']['data'], proc)
def test_sanitize_url(self): proc = SensitiveDataFilter() result = proc.sanitize('foo', 'pg://*****:*****@localhost/1') assert result == 'pg://*****:*****@localhost/1' % FILTER_MASK result = proc.sanitize('foo', "foo 'redis://*****:*****@localhost:6379/0' bar") assert result == "foo 'redis://*****:*****@localhost:6379/0' bar" % FILTER_MASK result = proc.sanitize('foo', "'redis://*****:*****@localhost:6379/0'") assert result == "'redis://*****:*****@localhost:6379/0'" % FILTER_MASK result = proc.sanitize('foo', "foo redis://redis:foo@localhost:6379/0 bar") assert result == "foo redis://redis:%s@localhost:6379/0 bar" % FILTER_MASK result = proc.sanitize( 'foo', "foo redis://redis:foo@localhost:6379/0 bar pg://matt:foo@localhost/1" ) assert result == "foo redis://redis:%s@localhost:6379/0 bar pg://matt:%s@localhost/1" % ( FILTER_MASK, FILTER_MASK ) # Make sure we don't mess up any other url. # This url specifically if passed through urlunsplit(urlsplit()), # it'll change the value. result = proc.sanitize('foo', 'postgres:///path') assert result == 'postgres:///path' # Don't be too overly eager within JSON strings an catch the right field. result = proc.sanitize( 'foo', '{"a":"https://localhost","b":"foo@localhost","c":"pg://*****:*****@localhost/1","d":"lol"}' ) assert result == '{"a":"https://localhost","b":"foo@localhost","c":"pg://*****:*****@localhost/1","d":"lol"}' % FILTER_MASK
def test_user_extra_data(self): """ Make sure we filter non-standard user entries, which all end up in user["data"] after normalization. See https://github.com/getsentry/semaphore/blob/5b6e46f6159843451027b5217902c81e134d7c40/general/src/protocol/user.rs#L54. """ data = {"user": {"data": VARS}} proc = SensitiveDataFilter() proc.apply(data) assert "user" in data self._check_vars_sanitized(data["user"]["data"], proc)
def test_user(self): data = { 'sentry.interfaces.User': { 'username': '******', 'data': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'sentry.interfaces.User' in data assert data['sentry.interfaces.User']['username'] == 'secret' self._check_vars_sanitized(data['sentry.interfaces.User']['data'], proc)
def test_csp_blocked_uri(self): data = { "csp": { "blocked_uri": "https://example.com/?foo=4571234567890111&bar=baz" } } proc = SensitiveDataFilter() proc.apply(data) assert "csp" in data csp = data["csp"] assert csp[ "blocked_uri"] == "https://example.com/?foo=[Filtered]&bar=baz"
def test_querystring_as_pairlist_with_partials(self): data = { "request": { "query_string": [["foo", "bar"], ["password", ""], ["baz", "bar"]] } } proc = SensitiveDataFilter() proc.apply(data) assert "request" in data http = data["request"] assert http["query_string"] == [["foo", "bar"], ["password", ""], ["baz", "bar"]]
def test_contexts(self): data = { 'contexts': { 'secret': VARS, 'biz': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'contexts' in data assert 'secret' in data['contexts'] assert 'biz' in data['contexts'] self._check_vars_sanitized(data['contexts']['secret'], proc) self._check_vars_sanitized(data['contexts']['biz'], proc)
def test_contexts(self): data = { 'contexts': { 'secret': VARS, 'biz': VARS, }, } proc = SensitiveDataFilter() proc.apply(data) assert 'contexts' in data assert 'secret' in data['contexts'] assert 'biz' in data['contexts'] self._check_vars_sanitized(data['contexts']['secret'], proc) self._check_vars_sanitized(data['contexts']['biz'], proc)
def test_querystring_as_string(self): data = { "request": { "query_string": "foo=bar&password=hello&the_secret=hello" "&a_password_here=hello&api_key=secret_key" } } proc = SensitiveDataFilter() proc.apply(data) self.assertTrue("request" in data) http = data["request"] self.assertEquals( http["query_string"], "foo=bar&password=%(m)s&the_secret=%(m)s" "&a_password_here=%(m)s&api_key=%(m)s" % dict(m=proc.MASK), )