def test_querystring_as_pairlist(self):
data = {
"request": {
"query_string": [
["foo", "bar"],
["password", "hello"],
["the_secret", "hello"],
["a_password_here", "hello"],
["api_key", "secret_key"],
]
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert "request" in data
http = data["request"]
assert http["query_string"] == [
["foo", "bar"],
["password", FILTER_MASK],
["the_secret", FILTER_MASK],
["a_password_here", FILTER_MASK],
["api_key", FILTER_MASK],
]
def test_querystring_as_pairlist(self):
data = {
'request': {
'query_string': [
['foo', 'bar'],
['password', 'hello'],
['the_secret', 'hello'],
['a_password_here', 'hello'],
['api_key', 'secret_key'],
],
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'request' in data
http = data['request']
assert http['query_string'] == [
['foo', 'bar'],
['password', FILTER_MASK],
['the_secret', FILTER_MASK],
['a_password_here', FILTER_MASK],
['api_key', FILTER_MASK],
]
def test_querystring_as_pairlist(self):
data = {
'request': {
'query_string': [
['foo', 'bar'],
['password', 'hello'],
['the_secret', 'hello'],
['a_password_here', 'hello'],
['api_key', 'secret_key'],
],
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'request' in data
http = data['request']
assert http['query_string'] == [
['foo', 'bar'],
['password', FILTER_MASK],
['the_secret', FILTER_MASK],
['a_password_here', FILTER_MASK],
['api_key', FILTER_MASK],
]
def test_extra(self):
data = {"extra": VARS}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue("extra" in data)
self._check_vars_sanitized(data["extra"], proc)
def test_extra(self):
data = {'extra': VARS}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('extra' in data)
self._check_vars_sanitized(data['extra'], proc)
def test_extra(self):
data = {"extra": VARS}
proc = SensitiveDataFilter()
proc.apply(data)
assert "extra" in data
self._check_vars_sanitized(data["extra"], proc)
def test_extra(self):
data = {'extra': VARS}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'extra' in data
self._check_vars_sanitized(data['extra'], proc)
def test_querystring_as_string_with_partials(self):
data = {"request": {"query_string": "foo=bar&password&baz=bar"}}
proc = SensitiveDataFilter()
proc.apply(data)
assert "request" in data
http = data["request"]
assert http["query_string"] == "foo=bar&password&baz=bar"
def test_querystring_as_string_with_partials(self):
data = {"request": {"query_string": "foo=bar&password&baz=bar"}}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue("request" in data)
http = data["request"]
self.assertEquals(http["query_string"], "foo=bar&password&baz=bar" % dict(m=proc.MASK))
def test_user(self):
data = {"user": {"username": "******", "data": VARS}}
proc = SensitiveDataFilter()
proc.apply(data)
assert "user" in data
assert data["user"]["username"] == "secret"
self._check_vars_sanitized(data["user"]["data"], proc)
def test_does_not_fail_on_non_string(self):
data = {
'extra': {
'foo': 1,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertEquals(data['extra'], {'foo': 1})
def test_empty_field(self):
data = {
'extra': {
'foobar': 'xxx',
},
}
proc = SensitiveDataFilter(fields=[''])
proc.apply(data)
assert data['extra'] == {'foobar': 'xxx'}
def test_does_sanitize_social_security_number(self):
data = {
'extra': {
's': '123-45-6789',
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'s': FILTER_MASK}
def test_does_sanitize_public_key(self):
data = {
'extra': {
's': PUBLIC_KEY,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'s': FILTER_MASK}
def test_explicit_fields(self):
data = {
'extra': {
'mystuff': 'xxx',
},
}
proc = SensitiveDataFilter(fields=['mystuff'])
proc.apply(data)
assert data['extra']['mystuff'] == FILTER_MASK
def test_exclude_fields_on_field_value(self):
data = {
'extra': {
'foobar': '123-45-6789',
},
}
proc = SensitiveDataFilter(exclude_fields=['foobar'])
proc.apply(data)
assert data['extra'] == {'foobar': '123-45-6789'}
def test_does_not_fail_on_non_string(self):
data = {
'extra': {
'foo': 1,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'foo': 1}
def test_exclude_fields_on_field_name(self):
data = {
'extra': {
'password': '******',
},
}
proc = SensitiveDataFilter(exclude_fields=['password'])
proc.apply(data)
assert data['extra'] == {'password': '******'}
def test_does_sanitize_social_security_number(self):
data = {
'extra': {
's': '123-45-6789',
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'s': FILTER_MASK}
def test_explicit_fields(self):
data = {
'extra': {
'mystuff': 'xxx',
},
}
proc = SensitiveDataFilter(fields=['mystuff'])
proc.apply(data)
assert data['extra']['mystuff'] == FILTER_MASK
def test_does_sanitize_rsa_private_key(self):
data = {
'extra': {
's': RSA_PRIVATE_KEY,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'s': FILTER_MASK}
def test_exclude_fields_on_field_name(self):
data = {
'extra': {
'password': '******',
},
}
proc = SensitiveDataFilter(exclude_fields=['password'])
proc.apply(data)
assert data['extra'] == {'password': '******'}
def test_does_sanitize_public_key(self):
data = {
'extra': {
's': PUBLIC_KEY,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'s': FILTER_MASK}
def test_empty_field(self):
data = {
'extra': {
'foobar': 'xxx',
},
}
proc = SensitiveDataFilter(fields=[''])
proc.apply(data)
assert data['extra'] == {'foobar': 'xxx'}
def test_exclude_fields_on_field_value(self):
data = {
'extra': {
'foobar': '123-45-6789',
},
}
proc = SensitiveDataFilter(exclude_fields=['foobar'])
proc.apply(data)
assert data['extra'] == {'foobar': '123-45-6789'}
def test_does_sanitize_rsa_private_key(self):
data = {
'extra': {
's': RSA_PRIVATE_KEY,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert data['extra'] == {'s': FILTER_MASK}
def process(self, request, project, auth, data, **kwargs):
event_received.send_robust(ip=request.META['REMOTE_ADDR'],
sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited, project=project)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
raise APIRateLimited(rate_limit.retry_after)
result = plugins.first('has_perm', request.user, 'create_event',
project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' %
(six.text_type(e), type(e)))
# mutates data
manager = EventManager(data)
data = manager.normalize()
# insert IP address if not available
if auth.is_public:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter()
inst.apply(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
logger.debug('New event from project %s/%s (id=%s)', project.team.slug,
project.slug, event_id)
return event_id
def test_http(self):
data = {"request": {"data": VARS, "env": VARS, "headers": VARS, "cookies": VARS}}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue("request" in data)
http = data["request"]
for n in ("data", "env", "headers", "cookies"):
self.assertTrue(n in http)
self._check_vars_sanitized(http[n], proc)
def test_sanitize_additional_sensitive_fields(self):
additional_sensitive_dict = {'fieldy_field': 'value', 'moar_other_field': 'another value'}
data = {'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))}
proc = SensitiveDataFilter(additional_sensitive_dict.keys())
proc.apply(data)
for field in additional_sensitive_dict.keys():
assert data['extra'][field] == FILTER_MASK
self._check_vars_sanitized(data['extra'], proc)
def test_sanitize_additional_sensitive_fields(self):
additional_sensitive_dict = {'fieldy_field': 'value', 'moar_other_field': 'another value'}
data = {'extra': dict(list(VARS.items()) + list(additional_sensitive_dict.items()))}
proc = SensitiveDataFilter(additional_sensitive_dict.keys())
proc.apply(data)
for field in additional_sensitive_dict.keys():
assert data['extra'][field] == FILTER_MASK
self._check_vars_sanitized(data['extra'], proc)
def test_contexts(self):
data = {"contexts": {"secret": VARS, "biz": VARS}}
proc = SensitiveDataFilter()
proc.apply(data)
assert "contexts" in data
assert "secret" in data["contexts"]
assert "biz" in data["contexts"]
self._check_vars_sanitized(data["contexts"]["secret"], proc)
self._check_vars_sanitized(data["contexts"]["biz"], proc)
def test_sanitize_http_body(self):
data = {
'sentry.interfaces.Http': {
'data': '{"email":"*****@*****.**","password":"******"}',
},
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Http' in data)
http = data['sentry.interfaces.Http']
self.assertEquals(http['data'], FILTER_MASK)
def test_sanitize_http_body(self):
data = {
"request": {
"data": '{"email":"*****@*****.**","password":"******"}'
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert "request" in data
http = data["request"]
assert http["data"] == FILTER_MASK
def test_sanitize_http_body(self):
data = {
'sentry.interfaces.Http': {
'data': '{"email":"*****@*****.**","password":"******"}',
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Http' in data
http = data['sentry.interfaces.Http']
assert http['data'] == FILTER_MASK
def test_sanitize_http_body(self):
data = {
'sentry.interfaces.Http': {
'data': '{"email":"*****@*****.**","password":"******"}',
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Http' in data
http = data['sentry.interfaces.Http']
assert http['data'] == FILTER_MASK
def test_sanitize_http_body(self):
data = {
'sentry.interfaces.Http': {
'data': '{"email":"*****@*****.**","password":"******"}',
},
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Http' in data)
http = data['sentry.interfaces.Http']
self.assertEquals(http['data'], FILTER_MASK)
def test_stacktrace(self):
data = {"stacktrace": {"frames": [{"vars": VARS}]}}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue("stacktrace" in data)
stack = data["stacktrace"]
self.assertTrue("frames" in stack)
self.assertEquals(len(stack["frames"]), 1)
frame = stack["frames"][0]
self.assertTrue("vars" in frame)
self._check_vars_sanitized(frame["vars"], proc)
def test_csp_blocked_uri(self):
data = {
'sentry.interfaces.Csp': {
'blocked_uri': 'https://example.com/?foo=4571234567890111&bar=baz',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Csp' in data
csp = data['sentry.interfaces.Csp']
assert csp['blocked_uri'] == 'https://example.com/?foo=[Filtered]&bar=baz'
def test_querystring_as_string_with_partials(self):
data = {
'sentry.interfaces.Http': {
'query_string': 'foo=bar&password&baz=bar',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Http' in data
http = data['sentry.interfaces.Http']
assert http['query_string'] == 'foo=bar&password&baz=bar'
def test_querystring_as_string_with_partials(self):
data = {
'sentry.interfaces.Http': {
'query_string': 'foo=bar&password&baz=bar',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Http' in data
http = data['sentry.interfaces.Http']
assert http['query_string'] == 'foo=bar&password&baz=bar'
def test_csp_blocked_uri(self):
data = {
'csp': {
'blocked_uri': 'https://example.com/?foo=4571234567890111&bar=baz',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'csp' in data
csp = data['csp']
assert csp['blocked_uri'] == 'https://example.com/?foo=[Filtered]&bar=baz'
def test_breadcrumb_message(self):
data = {
'breadcrumbs': {
'values': [{
'message': "SELECT session_key FROM django_session WHERE session_key = 'abcdefg'",
}],
},
}
proc = SensitiveDataFilter(fields=['session_key'])
proc.apply(data)
assert data['breadcrumbs']['values'][0]['message'] == FILTER_MASK
def test_querystring_as_string_with_partials(self):
data = {
'sentry.interfaces.Http': {
'query_string': 'foo=bar&password&baz=bar',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Http' in data)
http = data['sentry.interfaces.Http']
self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=FILTER_MASK))
def test_stacktrace(self):
data = {"stacktrace": {"frames": [{"vars": VARS}]}}
proc = SensitiveDataFilter()
proc.apply(data)
assert "stacktrace" in data
stack = data["stacktrace"]
assert "frames" in stack
assert len(stack["frames"]) == 1
frame = stack["frames"][0]
assert "vars" in frame
self._check_vars_sanitized(frame["vars"], proc)
def test_querystring_as_string_with_partials(self):
data = {
'sentry.interfaces.Http': {
'query_string': 'foo=bar&password&baz=bar',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Http' in data)
http = data['sentry.interfaces.Http']
self.assertEquals(http['query_string'], 'foo=bar&password&baz=bar' % dict(m=proc.MASK))
def process(self, request, project, auth, data, **kwargs):
event_received.send_robust(ip=request.META['REMOTE_ADDR'], sender=type(self))
# TODO: improve this API (e.g. make RateLimit act on __ne__)
rate_limit = safe_execute(app.quotas.is_rate_limited, project=project)
if isinstance(rate_limit, bool):
rate_limit = RateLimit(is_limited=rate_limit, retry_after=None)
if rate_limit is not None and rate_limit.is_limited:
raise APIRateLimited(rate_limit.retry_after)
result = plugins.first('has_perm', request.user, 'create_event', project)
if result is False:
raise APIForbidden('Creation of this event was blocked')
content_encoding = request.META.get('HTTP_CONTENT_ENCODING', '')
if content_encoding == 'gzip':
data = decompress_gzip(data)
elif content_encoding == 'deflate':
data = decompress_deflate(data)
elif not data.startswith('{'):
data = decode_and_decompress_data(data)
data = safely_load_json_string(data)
try:
# mutates data
validate_data(project, data, auth.client)
except InvalidData as e:
raise APIError(u'Invalid data: %s (%s)' % (six.text_type(e), type(e)))
# mutates data
manager = EventManager(data)
data = manager.normalize()
# insert IP address if not available
if auth.is_public:
ensure_has_ip(data, request.META['REMOTE_ADDR'])
event_id = data['event_id']
# We filter data immediately before it ever gets into the queue
inst = SensitiveDataFilter()
inst.apply(data)
# mutates data (strips a lot of context if not queued)
insert_data_to_database(data)
logger.debug('New event from project %s/%s (id=%s)', project.team.slug, project.slug, event_id)
return event_id
def test_querystring_as_string_with_partials(self):
data = {
'request': {
'query_string': 'foo=bar&password&baz=bar',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('request' in data)
http = data['request']
self.assertEquals(http['query_string'],
'foo=bar&password&baz=bar' % dict(m=proc.MASK))
def test_user(self):
data = {
'sentry.interfaces.User': {
'username': '******',
'data': VARS,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.User' in data
assert data['sentry.interfaces.User']['username'] == 'secret'
self._check_vars_sanitized(data['sentry.interfaces.User']['data'], proc)
def test_breadcrumb_message(self):
data = {
"breadcrumbs": {
"values": [{
"message":
"SELECT session_key FROM django_session WHERE session_key = 'abcdefg'"
}]
}
}
proc = SensitiveDataFilter(fields=["session_key"])
proc.apply(data)
assert data["breadcrumbs"]["values"][0]["message"] == FILTER_MASK
def test_user(self):
data = {
'sentry.interfaces.User': {
'username': '******',
'data': VARS,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.User' in data
assert data['sentry.interfaces.User']['username'] == 'secret'
self._check_vars_sanitized(data['sentry.interfaces.User']['data'], proc)
def test_user_extra_data(self):
"""
Make sure we filter non-standard user entries, which all end up in
user["data"] after normalization. See
https://github.com/getsentry/semaphore/blob/5b6e46f6159843451027b5217902c81e134d7c40/general/src/protocol/user.rs#L54.
"""
data = {"user": {"data": VARS}}
proc = SensitiveDataFilter()
proc.apply(data)
assert "user" in data
self._check_vars_sanitized(data["user"]["data"], proc)
def test_csp_blocked_uri(self):
data = {
"csp": {
"blocked_uri":
"https://example.com/?foo=4571234567890111&bar=baz"
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert "csp" in data
csp = data["csp"]
assert csp[
"blocked_uri"] == "https://example.com/?foo=[Filtered]&bar=baz"
def test_querystring_as_string(self):
data = {
"request": {
"query_string": "foo=bar&password=hello&the_secret=hello" "&a_password_here=hello&api_key=secret_key"
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue("request" in data)
http = data["request"]
self.assertEquals(
http["query_string"],
"foo=bar&password=%(m)s&the_secret=%(m)s" "&a_password_here=%(m)s&api_key=%(m)s" % dict(m=proc.MASK),
)
def test_contexts(self):
data = {
'contexts': {
'secret': VARS,
'biz': VARS,
},
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'contexts' in data
assert 'secret' in data['contexts']
assert 'biz' in data['contexts']
self._check_vars_sanitized(data['contexts']['secret'], proc)
self._check_vars_sanitized(data['contexts']['biz'], proc)
def test_querystring_as_string(self):
data = {
'sentry.interfaces.Http': {
'query_string': 'foo=bar&password=hello&the_secret=hello'
'&a_password_here=hello&api_key=secret_key',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Http' in data)
http = data['sentry.interfaces.Http']
self.assertEquals(
http['query_string'],
'foo=bar&password=%(m)s&the_secret=%(m)s'
'&a_password_here=%(m)s&api_key=%(m)s' % dict(m=proc.MASK))
def test_stacktrace(self):
data = {
'sentry.interfaces.Stacktrace': {
'frames': [{'vars': VARS}],
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Stacktrace' in data)
stack = data['sentry.interfaces.Stacktrace']
self.assertTrue('frames' in stack)
self.assertEquals(len(stack['frames']), 1)
frame = stack['frames'][0]
self.assertTrue('vars' in frame)
self._check_vars_sanitized(frame['vars'], proc)
def test_querystring_as_string(self):
data = {
'sentry.interfaces.Http': {
'query_string': 'foo=bar&password=hello&the_secret=hello'
'&a_password_here=hello&api_key=secret_key',
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Http' in data
http = data['sentry.interfaces.Http']
assert http['query_string'] == (
'foo=bar&password=%(m)s&the_secret=%(m)s'
'&a_password_here=%(m)s&api_key=%(m)s' % {'m': FILTER_MASK}
)
def test_stacktrace(self):
data = {
'sentry.interfaces.Stacktrace': {
'frames': [{'vars': VARS}],
}
}
proc = SensitiveDataFilter()
proc.apply(data)
assert 'sentry.interfaces.Stacktrace' in data
stack = data['sentry.interfaces.Stacktrace']
assert 'frames' in stack
assert len(stack['frames']) == 1
frame = stack['frames'][0]
assert 'vars' in frame
self._check_vars_sanitized(frame['vars'], proc)
def test_http(self):
data = {
'sentry.interfaces.Http': {
'data': VARS,
'env': VARS,
'headers': VARS,
'cookies': VARS,
}
}
proc = SensitiveDataFilter()
proc.apply(data)
self.assertTrue('sentry.interfaces.Http' in data)
http = data['sentry.interfaces.Http']
for n in ('data', 'env', 'headers', 'cookies'):
self.assertTrue(n in http)
self._check_vars_sanitized(http[n], proc)
