def validate_data(project, data, client=None):
ensure_valid_project_id(project, data, client=client)
if not data.get('message'):
data['message'] = '<no message value>'
elif not isinstance(data['message'], basestring):
raise APIError('Invalid value for message')
elif len(data['message']) > MAX_MESSAGE_LENGTH:
logger.info('Truncated value for message due to length (%d chars)',
len(data['message']), **client_metadata(client, project))
data['message'] = truncatechars(data['message'], MAX_MESSAGE_LENGTH)
if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH:
logger.info('Truncated value for culprit due to length (%d chars)',
len(data['culprit']), **client_metadata(client, project))
data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH)
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
if len(data['event_id']) > 32:
logger.info('Discarded value for event_id due to length (%d chars)',
len(data['event_id']), **client_metadata(client, project))
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp, e:
# Log the error, remove the timestamp, and continue
logger.info('Discarded invalid value for timestamp: %r',
data['timestamp'],
**client_metadata(client, project, exception=e))
del data['timestamp']
def validate_data(project, data, client=None):
ensure_valid_project_id(project, data, client=client)
if not data.get('message'):
data['message'] = '<no message value>'
elif not isinstance(data['message'], basestring):
raise APIError('Invalid value for message')
elif len(data['message']) > MAX_MESSAGE_LENGTH:
logger.info('Truncated value for message due to length (%d chars)', len(data['message']),
**client_metadata(client))
data['message'] = truncatechars(data['message'], MAX_MESSAGE_LENGTH)
if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH:
logger.info('Truncated value for culprit due to length (%d chars)', len(data['culprit']),
**client_metadata(client))
data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH)
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
if len(data['event_id']) > 32:
logger.info('Discarded value for event_id due to length (%d chars)', len(data['event_id']),
**client_metadata(client))
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp, e:
# Log the error, remove the timestamp, and continue
logger.info('Discarded invalid value for timestamp: %r', data['timestamp'],
**client_metadata(client, exception=e))
del data['timestamp']
def message_short(self):
message = strip(self.message)
if not message:
message = '<unlabeled message>'
else:
message = truncatechars(message.splitlines()[0], 100)
return message
def generate_culprit(data):
platform = data.get('platform')
exceptions = get_path(data, 'exception', 'values', filter=True)
if exceptions:
# Synthetic events no longer get a culprit
last_exception = get_path(exceptions, -1)
if get_path(last_exception, 'mechanism', 'synthetic'):
return ''
stacktraces = [e['stacktrace'] for e in exceptions if get_path(e, 'stacktrace', 'frames')]
else:
stacktrace = data.get('stacktrace')
if stacktrace and stacktrace.get('frames'):
stacktraces = [stacktrace]
else:
stacktraces = None
culprit = None
if not culprit and stacktraces:
culprit = get_stacktrace_culprit(get_path(stacktraces, -1), platform=platform)
if not culprit and data.get('request'):
culprit = get_path(data, 'request', 'url')
return truncatechars(culprit or '', MAX_CULPRIT_LENGTH)
def generate_culprit(data):
platform = data.get('platform')
exceptions = get_path(data, 'exception', 'values', filter=True)
if exceptions:
# Synthetic events no longer get a culprit
last_exception = get_path(exceptions, -1)
if get_path(last_exception, 'mechanism', 'synthetic'):
return ''
stacktraces = [
e['stacktrace'] for e in exceptions
if get_path(e, 'stacktrace', 'frames')
]
else:
stacktrace = data.get('stacktrace')
if stacktrace and stacktrace.get('frames'):
stacktraces = [stacktrace]
else:
stacktraces = None
culprit = None
if not culprit and stacktraces:
culprit = get_stacktrace_culprit(get_path(stacktraces, -1),
platform=platform)
if not culprit and data.get('request'):
culprit = get_path(data, 'request', 'url')
return truncatechars(culprit or '', MAX_CULPRIT_LENGTH)
def test_truncated(self):
url = truncatechars('http://example.com', 3)
with pytest.raises(CannotFetchSource) as exc:
fetch_file(url)
assert exc.value.data['type'] == EventError.JS_MISSING_SOURCE
assert exc.value.data['url'] == url
def error(self):
message = strip(self.message)
if message:
message = truncatechars(message, 100)
else:
message = '<unlabeled message>'
return message
def generate_culprit(data, platform=None):
culprit = ''
try:
stacktraces = [
e['stacktrace'] for e in data['sentry.interfaces.Exception']['values']
if e.get('stacktrace')
]
except KeyError:
stacktrace = data.get('sentry.interfaces.Stacktrace')
if stacktrace:
stacktraces = [stacktrace]
else:
stacktraces = None
if not stacktraces:
if 'sentry.interfaces.Http' in data:
culprit = data['sentry.interfaces.Http'].get('url', '')
else:
from sentry.interfaces.stacktrace import Stacktrace
culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string(
platform=platform,
)
return truncatechars(culprit, MAX_CULPRIT_LENGTH)
def error(self):
message = strip(self.message)
if message:
message = truncatechars(message, 100)
else:
message = '<unlabeled message>'
return message
def test_truncated(self):
url = truncatechars('http://example.com', 3)
with pytest.raises(http.CannotFetch) as exc:
fetch_file(url)
assert exc.value.data['type'] == EventError.JS_MISSING_SOURCE
assert exc.value.data['url'] == url
def get_email_subject(self):
template = self.project.get_option("mail:subject_template")
if template:
template = EventSubjectTemplate(template)
else:
template = DEFAULT_SUBJECT_TEMPLATE
return truncatechars(template.safe_substitute(EventSubjectTemplateData(self)), 128)
def to_string(self, metadata):
if not metadata['value']:
return metadata['type']
return u'{}: {}'.format(
metadata['type'],
truncatechars(metadata['value'].splitlines()[0], 100),
)
def error(self):
message = strip(self.get_legacy_message())
if not message:
message = '<unlabeled message>'
else:
message = truncatechars(message.splitlines()[0], 100)
return message
def message_short(self):
message = strip(self.message)
if not message:
message = '<unlabeled message>'
else:
message = truncatechars(message.splitlines()[0], 100)
return message
def to_string(self, metadata):
if not metadata['value']:
return metadata['type']
return u'{}: {}'.format(
metadata['type'],
truncatechars(metadata['value'].splitlines()[0], 100),
)
def generate_culprit(data):
platform = data.get("platform")
exceptions = get_path(data, "exception", "values", filter=True)
if exceptions:
# Synthetic events no longer get a culprit
last_exception = get_path(exceptions, -1)
if get_path(last_exception, "mechanism", "synthetic"):
return ""
stacktraces = [
e["stacktrace"] for e in exceptions
if get_path(e, "stacktrace", "frames")
]
else:
stacktrace = data.get("stacktrace")
if stacktrace and stacktrace.get("frames"):
stacktraces = [stacktrace]
else:
stacktraces = None
culprit = None
if not culprit and stacktraces:
culprit = get_stacktrace_culprit(get_path(stacktraces, -1),
platform=platform)
if not culprit and data.get("request"):
culprit = get_path(data, "request", "url")
return truncatechars(culprit or "", MAX_CULPRIT_LENGTH)
def generate_culprit(data, platform=None):
culprit = ''
try:
stacktraces = [
e['stacktrace']
for e in data['sentry.interfaces.Exception']['values']
if e.get('stacktrace')
]
except KeyError:
stacktrace = data.get('sentry.interfaces.Stacktrace')
if stacktrace:
stacktraces = [stacktrace]
else:
stacktraces = None
if not stacktraces:
if 'sentry.interfaces.Http' in data:
culprit = data['sentry.interfaces.Http'].get('url', '')
else:
from sentry.interfaces.stacktrace import Stacktrace
culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string(
platform=platform,
)
return truncatechars(culprit, MAX_CULPRIT_LENGTH)
def error(self):
message = strip(self.get_legacy_message())
if not message:
message = '<unlabeled message>'
else:
message = truncatechars(message.splitlines()[0], 100)
return message
def get_title(self, metadata):
ty = metadata.get("type")
if ty is None:
return metadata.get("function") or "<unknown>"
if not metadata.get("value"):
return ty
return u"{}: {}".format(ty, truncatechars(metadata["value"].splitlines()[0], 100))
def generate_culprit(data, platform=None):
exceptions = get_path(data, 'exception', 'values')
if exceptions:
stacktraces = [
e['stacktrace'] for e in exceptions
if get_path(e, 'stacktrace', 'frames')
]
else:
stacktrace = data.get('stacktrace')
if stacktrace and stacktrace.get('frames'):
stacktraces = [stacktrace]
else:
stacktraces = None
culprit = None
if not culprit and stacktraces:
from sentry.interfaces.stacktrace import Stacktrace
culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string(
platform=platform, )
if not culprit and data.get('request'):
culprit = get_path(data, 'request', 'url')
return truncatechars(culprit or '', MAX_CULPRIT_LENGTH)
def trim(
value,
max_size=settings.SENTRY_MAX_VARIABLE_SIZE,
max_depth=6,
object_hook=None,
_depth=0,
_size=0,
**kwargs,
):
"""
Truncates a value to ```MAX_VARIABLE_SIZE```.
The method of truncation depends on the type of value.
"""
options = {
"max_depth": max_depth,
"max_size": max_size,
"object_hook": object_hook,
"_depth": _depth + 1,
}
if _depth > max_depth:
if not isinstance(value, str):
value = json.dumps(value)
return trim(value, _size=_size, max_size=max_size)
elif isinstance(value, dict):
result = {}
_size += 2
for k in sorted(value.keys(),
key=lambda x: (len(force_text(value[x])), x)):
v = value[k]
trim_v = trim(v, _size=_size, **options)
result[k] = trim_v
_size += len(force_text(trim_v)) + 1
if _size >= max_size:
break
elif isinstance(value, (list, tuple)):
result = []
_size += 2
for v in value:
trim_v = trim(v, _size=_size, **options)
result.append(trim_v)
_size += len(force_text(trim_v))
if _size >= max_size:
break
if isinstance(value, tuple):
result = tuple(result)
elif isinstance(value, str):
result = truncatechars(value, max_size - _size)
else:
result = value
if object_hook is None:
return result
return object_hook(result)
def message_top(self):
culprit = strip(self.culprit)
if culprit:
return culprit
message = strip(self.message)
if not strip(message):
return '<unlabeled message>'
return truncatechars(message.splitlines()[0], 100)
def message_top(self):
culprit = strip(self.culprit)
if culprit:
return culprit
message = strip(self.message)
if not strip(message):
return '<unlabeled message>'
return truncatechars(message.splitlines()[0], 100)
def trim(
value,
max_size=settings.SENTRY_MAX_VARIABLE_SIZE,
max_depth=6,
object_hook=None,
_depth=0,
_size=0,
**kwargs
):
"""
Truncates a value to ```MAX_VARIABLE_SIZE```.
The method of truncation depends on the type of value.
"""
options = {
'max_depth': max_depth,
'max_size': max_size,
'object_hook': object_hook,
'_depth': _depth + 1,
}
if _depth > max_depth and ENABLE_TRIMMING:
if not isinstance(value, six.string_types):
value = json.dumps(value)
return trim(value, _size=_size, max_size=max_size)
elif isinstance(value, dict):
result = {}
_size += 2
for k in sorted(value.keys()):
v = value[k]
trim_v = trim(v, _size=_size, **options)
result[k] = trim_v
_size += len(force_text(trim_v)) + 1
if _size >= max_size and ENABLE_TRIMMING:
break
elif isinstance(value, (list, tuple)):
result = []
_size += 2
for v in value:
trim_v = trim(v, _size=_size, **options)
result.append(trim_v)
_size += len(force_text(trim_v))
if _size >= max_size and ENABLE_TRIMMING:
break
if isinstance(value, tuple):
result = tuple(result)
elif isinstance(value, six.string_types) and ENABLE_TRIMMING:
result = truncatechars(value, max_size - _size)
else:
result = value
if object_hook is None:
return result
return object_hook(result)
def get_metadata(self):
message = strip(self.data.get('message'))
if not message:
title = '<unlabeled event>'
else:
title = truncatechars(message.splitlines()[0], 100)
return {
'title': title,
}
def expose_url(url):
if url is None:
return u'<unknown>'
if url[:5] == 'data:':
return u'<data url>'
url = truncatechars(url, MAX_URL_LENGTH)
if isinstance(url, bytes):
url = url.decode('utf-8', 'replace')
return url
def expose_url(url):
if url is None:
return "<unknown>"
if url[:5] == "data:":
return "<data url>"
url = truncatechars(url, MAX_URL_LENGTH)
if isinstance(url, six.binary_type):
url = url.decode("utf-8", "replace")
return url
def expose_url(url):
if url is None:
return u'<unknown>'
if url[:5] == 'data:':
return u'<data url>'
url = truncatechars(url, MAX_URL_LENGTH)
if isinstance(url, six.binary_type):
url = url.decode('utf-8', 'replace')
return url
def validate_data(project, data, client=None):
ensure_valid_project_id(project, data, client=client)
if not data.get("message"):
data["message"] = "<no message value>"
elif not isinstance(data["message"], basestring):
raise APIError("Invalid value for message")
elif len(data["message"]) > settings.SENTRY_MAX_MESSAGE_LENGTH:
logger.info(
"Truncated value for message due to length (%d chars)",
len(data["message"]),
**client_metadata(client, project)
)
data["message"] = truncatechars(data["message"], settings.SENTRY_MAX_MESSAGE_LENGTH)
if data.get("culprit") and len(data["culprit"]) > MAX_CULPRIT_LENGTH:
logger.info(
"Truncated value for culprit due to length (%d chars)",
len(data["culprit"]),
**client_metadata(client, project)
)
data["culprit"] = truncatechars(data["culprit"], MAX_CULPRIT_LENGTH)
if not data.get("event_id"):
data["event_id"] = uuid.uuid4().hex
if len(data["event_id"]) > 32:
logger.info(
"Discarded value for event_id due to length (%d chars)",
len(data["event_id"]),
**client_metadata(client, project)
)
data["event_id"] = uuid.uuid4().hex
if "timestamp" in data:
try:
process_data_timestamp(data)
except InvalidTimestamp, e:
# Log the error, remove the timestamp, and continue
logger.info(
"Discarded invalid value for timestamp: %r",
data["timestamp"],
**client_metadata(client, project, exception=e)
)
del data["timestamp"]
def trim(
value,
max_size=settings.SENTRY_MAX_VARIABLE_SIZE,
max_depth=6,
object_hook=None,
_depth=0,
_size=0,
**kwargs
):
"""
Truncates a value to ```MAX_VARIABLE_SIZE```.
The method of truncation depends on the type of value.
"""
options = {
'max_depth': max_depth,
'max_size': max_size,
'object_hook': object_hook,
'_depth': _depth + 1,
}
if _depth > max_depth:
if not isinstance(value, six.string_types):
value = json.dumps(value)
return trim(value, _size=_size, max_size=max_size)
elif isinstance(value, dict):
result = {}
_size += 2
for k in sorted(value.keys()):
v = value[k]
trim_v = trim(v, _size=_size, **options)
result[k] = trim_v
_size += len(force_text(trim_v)) + 1
if _size >= max_size:
break
elif isinstance(value, (list, tuple)):
result = []
_size += 2
for v in value:
trim_v = trim(v, _size=_size, **options)
result.append(trim_v)
_size += len(force_text(trim_v))
if _size >= max_size:
break
elif isinstance(value, six.string_types):
result = truncatechars(value, max_size - _size)
else:
result = value
if object_hook is None:
return result
return object_hook(result)
def fix_culprit(self, data, stacktraces):
# This is a bit weird, since the original culprit we get
# will be wrong, so we want to touch it up after we've processed
# a stack trace.
# In this case, we have a list of all stacktraces as a tuple
# (stacktrace as dict, stacktrace class)
# So we need to take the [1] index to get the Stacktrace class,
# then extract the culprit string from that.
data["culprit"] = truncatechars(stacktraces[-1][1].get_culprit_string(), MAX_CULPRIT_LENGTH)
def get_title(self, metadata):
ty = metadata.get('type')
if ty is None:
return metadata.get('function') or '<unknown>'
if not metadata.get('value'):
return ty
return u'{}: {}'.format(
ty,
truncatechars(metadata['value'].splitlines()[0], 100),
)
def extract_metadata(self, data):
message = strip(
get_path(data, "logentry", "formatted")
or get_path(data, "logentry", "message"))
if message:
title = truncatechars(message.splitlines()[0], 100)
else:
title = "<unlabeled event>"
return {"title": title}
def get_email_subject(self):
template = self.project.get_option('mail:subject_template')
if template:
template = EventSubjectTemplate(template)
else:
template = DEFAULT_SUBJECT_TEMPLATE
return truncatechars(
template.safe_substitute(
EventSubjectTemplateData(self),
),
128,
).encode('utf-8')
def save(self, *args, **kwargs):
if not self.last_seen:
self.last_seen = timezone.now()
if not self.first_seen:
self.first_seen = self.last_seen
if not self.active_at:
self.active_at = self.first_seen
# We limit what we store for the message body
self.message = strip(self.message)
if self.message:
self.message = truncatechars(self.message.splitlines()[0], 255)
super(Group, self).save(*args, **kwargs)
def get_metadata(self):
message = strip(get_path(self.data, 'logentry', 'formatted') or
get_path(self.data, 'logentry', 'message'))
if message:
title = truncatechars(message.splitlines()[0], 100)
else:
title = '<unlabeled event>'
return {
'title': title,
}
def save(self, *args, **kwargs):
if not self.last_seen:
self.last_seen = timezone.now()
if not self.first_seen:
self.first_seen = self.last_seen
if not self.active_at:
self.active_at = self.first_seen
# We limit what we store for the message body
self.message = strip(self.message)
if self.message:
self.message = truncatechars(self.message.splitlines()[0], 255)
super(Group, self).save(*args, **kwargs)
def get_metadata(self, data):
message = strip(get_path(data, 'logentry', 'formatted') or
get_path(data, 'logentry', 'message'))
if message:
title = truncatechars(message.splitlines()[0], 100)
else:
title = '<unlabeled event>'
return {
'title': title,
}
def get_metadata(self):
# See GH-3248
message_interface = self.data.get('sentry.interfaces.Message', {
'message': self.data.get('message', ''),
})
message = strip(message_interface.get('formatted', message_interface['message']))
if not message:
title = '<unlabeled event>'
else:
title = truncatechars(message.splitlines()[0], 100)
return {
'title': title,
}
def fix_culprit(self, data, stacktraces):
# This is a bit weird, since the original culprit we get
# will be wrong, so we want to touch it up after we've processed
# a stack trace.
# In this case, we have a list of all stacktraces as a tuple
# (stacktrace as dict, stacktrace class)
# So we need to take the [1] index to get the Stacktrace class,
# then extract the culprit string from that.
data['culprit'] = truncatechars(
stacktraces[-1][1].get_culprit_string(),
MAX_CULPRIT_LENGTH,
)
def get_metadata(self):
# See GH-3248
message_interface = self.data.get('sentry.interfaces.Message', {
'message': self.data.get('message', ''),
})
message = strip(message_interface.get('formatted', message_interface['message']))
if not message:
title = '<unlabeled event>'
else:
title = truncatechars(message.splitlines()[0], 100)
return {
'title': title,
}
def trim(value,
max_size=settings.SENTRY_MAX_VARIABLE_SIZE,
max_depth=3,
object_hook=None,
_depth=0,
_size=0,
**kwargs):
"""
Truncates a value to ```MAX_VARIABLE_SIZE```.
The method of truncation depends on the type of value.
"""
options = {
'max_depth': max_depth,
'max_size': max_size,
'object_hook': object_hook,
'_depth': _depth + 1,
}
if _depth > max_depth:
return trim(repr(value), _size=_size, max_size=max_size)
elif isinstance(value, dict):
result = {}
_size += 2
for k, v in value.iteritems():
trim_v = trim(v, _size=_size, **options)
result[k] = trim_v
_size += len(six.text_type(trim_v)) + 1
if _size >= max_size:
break
elif isinstance(value, (list, tuple)):
result = []
_size += 2
for v in value:
trim_v = trim(v, _size=_size, **options)
result.append(trim_v)
_size += len(six.text_type(trim_v))
if _size >= max_size:
break
elif isinstance(value, six.string_types):
result = truncatechars(value, max_size - _size)
else:
result = value
if object_hook is None:
return result
return object_hook(result)
def save(self, *args, **kwargs):
if not self.last_seen:
self.last_seen = timezone.now()
if not self.first_seen:
self.first_seen = self.last_seen
if not self.active_at:
self.active_at = self.first_seen
# We limit what we store for the message body
self.message = strip(self.message)
if self.message:
self.message = truncatechars(self.message.splitlines()[0], 255)
if self.times_seen is None:
self.times_seen = 1
self.score = type(self).calculate_score(times_seen=self.times_seen,
last_seen=self.last_seen)
super().save(*args, **kwargs)
def trim(value,
max_size=MAX_VARIABLE_SIZE,
max_depth=3,
_depth=0,
_size=0,
**kwargs):
"""
Truncates a value to ```MAX_VARIABLE_SIZE```.
The method of truncation depends on the type of value.
"""
options = {
'max_depth': max_depth,
'max_size': max_size,
'_depth': _depth + 1,
}
if _depth > max_depth:
return trim(repr(value), _size=_size, max_size=max_size)
elif isinstance(value, dict):
result = {}
_size += 2
for k, v in value.iteritems():
trim_v = trim(v, _size=_size, **options)
result[k] = trim_v
_size += len(unicode(trim_v)) + 1
if _size >= max_size:
break
elif isinstance(value, (list, tuple)):
result = []
_size += 2
for v in value:
trim_v = trim(v, _size=_size, **options)
result.append(trim_v)
_size += len(unicode(trim_v))
if _size >= max_size:
break
elif isinstance(value, basestring):
result = truncatechars(value, max_size - _size)
else:
result = value
return result
def to_string(self, metadata):
if metadata.get('message'):
return metadata['message']
if metadata.get('title'):
return metadata['title']
if metadata.get('type') and metadata.get('value'):
return u'{}: {}'.format(
metadata['type'],
truncatechars(metadata['value'].splitlines()[0], 100),
)
if metadata.get('type'):
return metadata['type']
return '<unlabeled event>'
def trim(value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=3,
_depth=0, _size=0, **kwargs):
"""
Truncates a value to ```MAX_VARIABLE_SIZE```.
The method of truncation depends on the type of value.
"""
options = {
'max_depth': max_depth,
'max_size': max_size,
'_depth': _depth + 1,
}
if _depth > max_depth:
return trim(repr(value), _size=_size, max_size=max_size)
elif isinstance(value, dict):
result = {}
_size += 2
for k, v in value.iteritems():
trim_v = trim(v, _size=_size, **options)
result[k] = trim_v
_size += len(unicode(trim_v)) + 1
if _size >= max_size:
break
elif isinstance(value, (list, tuple)):
result = []
_size += 2
for v in value:
trim_v = trim(v, _size=_size, **options)
result.append(trim_v)
_size += len(unicode(trim_v))
if _size >= max_size:
break
elif isinstance(value, basestring):
result = truncatechars(value, max_size - _size)
else:
result = value
return result
def generate_culprit(data, platform=None):
culprit = ""
try:
stacktraces = [e["stacktrace"] for e in data["sentry.interfaces.Exception"]["values"] if e.get("stacktrace")]
except KeyError:
if "sentry.interfaces.Stacktrace" in data:
stacktraces = [data["sentry.interfaces.Stacktrace"]]
else:
stacktraces = None
if not stacktraces:
if "sentry.interfaces.Http" in data:
culprit = data["sentry.interfaces.Http"].get("url", "")
else:
from sentry.interfaces.stacktrace import Stacktrace
culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string(platform=platform)
return truncatechars(culprit, MAX_CULPRIT_LENGTH)
def test_long_email(self):
org = self.create_organization()
project = self.create_project(organization=org, name="foo")
repo = Repository.objects.create(organization_id=org.id,
name="test/repo")
release = Release.objects.create(version="abcdabc", organization=org)
release.add_project(project)
commit_email = "a" * 248 + "@a.com" # 254 chars long, max valid email.
release.set_commits([{
"id": "a" * 40,
"repository": repo.name,
"author_name": "foo bar baz",
"author_email": commit_email,
"message": "i fixed a bug",
}])
commit = Commit.objects.get(repository_id=repo.id,
organization_id=org.id,
key="a" * 40)
assert commit.author.email == truncatechars(commit_email, 75)
def generate_culprit(data):
from sentry.interfaces.stacktrace import Stacktrace
try:
stacktraces = [
e['stacktrace']
for e in data['sentry.interfaces.Exception']['values']
if e.get('stacktrace')
]
except KeyError:
if 'sentry.interfaces.Stacktrace' in data:
stacktraces = [data['sentry.interfaces.Stacktrace']]
else:
return ''
if not stacktraces:
return ''
return truncatechars(
Stacktrace.to_python(stacktraces[-1]).get_culprit_string(),
MAX_CULPRIT_LENGTH
)
def generate_culprit(data):
from sentry.interfaces.stacktrace import Stacktrace
try:
stacktraces = [
e['stacktrace']
for e in data['sentry.interfaces.Exception']['values']
if e.get('stacktrace')
]
except KeyError:
if 'sentry.interfaces.Stacktrace' in data:
stacktraces = [data['sentry.interfaces.Stacktrace']]
else:
return ''
if not stacktraces:
return ''
return truncatechars(
Stacktrace.to_python(stacktraces[-1]).get_culprit_string(),
MAX_CULPRIT_LENGTH
)
def validate_data(project, data, client=None):
# TODO(dcramer): move project out of the data packet
data['project'] = project.id
if not data.get('message'):
data['message'] = '<no message value>'
elif not isinstance(data['message'], six.string_types):
raise APIError('Invalid value for message')
elif len(data['message']) > settings.SENTRY_MAX_MESSAGE_LENGTH:
logger.info(
'Truncated value for message due to length (%d chars)',
len(data['message']), **client_metadata(client, project))
data['message'] = truncatechars(
data['message'], settings.SENTRY_MAX_MESSAGE_LENGTH)
if data.get('culprit'):
if not isinstance(data['culprit'], six.string_types):
raise APIError('Invalid value for culprit')
logger.info(
'Truncated value for culprit due to length (%d chars)',
len(data['culprit']), **client_metadata(client, project))
data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH)
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif not isinstance(data['event_id'], six.string_types):
raise APIError('Invalid value for event_id')
if len(data['event_id']) > 32:
logger.info(
'Discarded value for event_id due to length (%d chars)',
len(data['event_id']), **client_metadata(client, project))
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp as e:
# Log the error, remove the timestamp, and continue
logger.info(
'Discarded invalid value for timestamp: %r', data['timestamp'],
**client_metadata(client, project, exception=e))
del data['timestamp']
if data.get('modules') and type(data['modules']) != dict:
logger.info(
'Discarded invalid type for modules: %s',
type(data['modules']), **client_metadata(client, project))
del data['modules']
if data.get('extra') is not None and type(data['extra']) != dict:
logger.info(
'Discarded invalid type for extra: %s',
type(data['extra']), **client_metadata(client, project))
del data['extra']
if data.get('tags') is not None:
if type(data['tags']) == dict:
data['tags'] = data['tags'].items()
elif not isinstance(data['tags'], (list, tuple)):
logger.info(
'Discarded invalid type for tags: %s',
type(data['tags']), **client_metadata(client, project))
del data['tags']
if data.get('tags'):
# remove any values which are over 32 characters
tags = []
for pair in data['tags']:
try:
k, v = pair
except ValueError:
logger.info('Discarded invalid tag value: %r',
pair, **client_metadata(client, project))
continue
if not isinstance(k, six.string_types):
try:
k = six.text_type(k)
except Exception:
logger.info('Discarded invalid tag key: %r',
type(k), **client_metadata(client, project))
continue
if not isinstance(v, six.string_types):
try:
v = six.text_type(v)
except Exception:
logger.info('Discarded invalid tag value: %s=%r',
k, type(v), **client_metadata(client, project))
continue
if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH:
logger.info('Discarded invalid tag: %s=%s',
k, v, **client_metadata(client, project))
continue
tags.append((k, v))
data['tags'] = tags
for k in data.keys():
if k in RESERVED_FIELDS:
continue
value = data.pop(k)
if not value:
logger.info(
'Ignored empty interface value: %s', k,
**client_metadata(client, project))
continue
try:
interface = get_interface(k)
except ValueError:
logger.info(
'Ignored unknown attribute: %s', k,
**client_metadata(client, project))
continue
if type(value) != dict:
# HACK(dcramer): the exception interface supports a list as the
# value. We should change this in a new protocol version.
if type(value) in (list, tuple):
value = {'values': value}
else:
logger.info(
'Invalid parameters for value: %s', k,
type(value), **client_metadata(client, project))
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, AssertionError):
log = logger.info
else:
log = logger.error
log('Discarded invalid value for interface: %s', k,
**client_metadata(client, project, exception=e, extra={'value': value}))
level = data.get('level') or DEFAULT_LOG_LEVEL
if isinstance(level, six.string_types) and not level.isdigit():
# assume it's something like 'warning'
try:
data['level'] = LOG_LEVEL_REVERSE_MAP[level]
except KeyError as e:
logger.info(
'Discarded invalid logger value: %s', level,
**client_metadata(client, project, exception=e))
data['level'] = LOG_LEVEL_REVERSE_MAP.get(
DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL)
return data
def test_truncatechars():
assert truncatechars("12345", 6) == "12345"
assert truncatechars("12345", 5) == "12345"
assert truncatechars("12345", 4) == "1..."
assert truncatechars("12345", 3) == "..."
assert truncatechars("12345", 2) == "..."
assert truncatechars("12345", 1) == "..."
assert truncatechars("12345", 0) == "..."
assert truncatechars("12345", 6, ellipsis=u"\u2026") == u"12345"
assert truncatechars("12345", 5, ellipsis=u"\u2026") == u"12345"
assert truncatechars("12345", 4, ellipsis=u"\u2026") == u"123\u2026"
assert truncatechars("12345", 3, ellipsis=u"\u2026") == u"12\u2026"
assert truncatechars("12345", 2, ellipsis=u"\u2026") == u"1\u2026"
assert truncatechars("12345", 1, ellipsis=u"\u2026") == u"\u2026"
assert truncatechars("12345", 0, ellipsis=u"\u2026") == u"\u2026"
assert truncatechars(None, 1) is None
def get_note(self):
if self.event == AuditLogEntryEvent.MEMBER_INVITE:
return 'invited member %s' % (self.data['email'], )
elif self.event == AuditLogEntryEvent.MEMBER_ADD:
if self.target_user == self.actor:
return 'joined the organization'
return 'added member %s' % (self.target_user.get_display_name(), )
elif self.event == AuditLogEntryEvent.MEMBER_ACCEPT:
return 'accepted the membership invite'
elif self.event == AuditLogEntryEvent.MEMBER_REMOVE:
if self.target_user == self.actor:
return 'left the organization'
return 'removed member %s' % (
self.data.get('email') or self.target_user.get_display_name(),
)
elif self.event == AuditLogEntryEvent.MEMBER_EDIT:
return 'edited member %s (role: %s, teams: %s)' % (
self.data.get('email') or self.target_user.get_display_name(),
self.data.get('role') or 'N/A',
', '.join(six.text_type(x) for x in self.data.get('team_slugs', [])) or 'N/A',
)
elif self.event == AuditLogEntryEvent.MEMBER_JOIN_TEAM:
if self.target_user == self.actor:
return 'joined team %s' % (self.data['team_slug'], )
return 'added %s to team %s' % (
self.data.get('email') or self.target_user.get_display_name(),
self.data['team_slug'],
)
elif self.event == AuditLogEntryEvent.MEMBER_LEAVE_TEAM:
if self.target_user == self.actor:
return 'left team %s' % (self.data['team_slug'], )
return 'removed %s from team %s' % (
self.data.get('email') or self.target_user.get_display_name(),
self.data['team_slug'],
)
elif self.event == AuditLogEntryEvent.MEMBER_PENDING:
return 'required member %s to setup 2FA' % (
self.data.get('email') or self.target_user.get_display_name(),
)
elif self.event == AuditLogEntryEvent.ORG_ADD:
return 'created the organization'
elif self.event == AuditLogEntryEvent.ORG_EDIT:
return 'edited the organization setting: ' + (', '.join(u'{} {}'.format(k, v)
for k, v in self.data.items()))
elif self.event == AuditLogEntryEvent.ORG_REMOVE:
return 'removed the organization'
elif self.event == AuditLogEntryEvent.ORG_RESTORE:
return 'restored the organization'
elif self.event == AuditLogEntryEvent.TEAM_ADD:
return 'created team %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.TEAM_EDIT:
return 'edited team %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.TEAM_REMOVE:
return 'removed team %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.PROJECT_ADD:
return 'created project %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.PROJECT_EDIT:
return 'edited project settings ' + (' '.join([' in %s to %s' % (key, value)
for (key, value) in six.iteritems(self.data)]))
elif self.event == AuditLogEntryEvent.PROJECT_REMOVE:
return 'removed project %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.PROJECT_REQUEST_TRANSFER:
return 'requested to transfer project %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.PROJECT_ACCEPT_TRANSFER:
return 'accepted transfer of project %s' % (self.data['slug'], )
elif self.event == AuditLogEntryEvent.TAGKEY_REMOVE:
return 'removed tags matching %s = *' % (self.data['key'], )
elif self.event == AuditLogEntryEvent.PROJECTKEY_ADD:
return 'added project key %s' % (self.data['public_key'], )
elif self.event == AuditLogEntryEvent.PROJECTKEY_EDIT:
return 'edited project key %s' % (self.data['public_key'], )
elif self.event == AuditLogEntryEvent.PROJECTKEY_REMOVE:
return 'removed project key %s' % (self.data['public_key'], )
elif self.event == AuditLogEntryEvent.PROJECTKEY_ENABLE:
return 'enabled project key %s' % (self.data['public_key'], )
elif self.event == AuditLogEntryEvent.PROJECTKEY_DISABLE:
return 'disabled project key %s' % (self.data['public_key'], )
elif self.event == AuditLogEntryEvent.SSO_ENABLE:
return 'enabled sso (%s)' % (self.data['provider'], )
elif self.event == AuditLogEntryEvent.SSO_DISABLE:
return 'disabled sso (%s)' % (self.data['provider'], )
elif self.event == AuditLogEntryEvent.SSO_EDIT:
return 'edited sso settings'
elif self.event == AuditLogEntryEvent.SSO_IDENTITY_LINK:
return 'linked their account to a new identity'
elif self.event == AuditLogEntryEvent.APIKEY_ADD:
return 'added api key %s' % (self.data['label'], )
elif self.event == AuditLogEntryEvent.APIKEY_EDIT:
return 'edited api key %s' % (self.data['label'], )
elif self.event == AuditLogEntryEvent.APIKEY_REMOVE:
return 'removed api key %s' % (self.data['label'], )
elif self.event == AuditLogEntryEvent.RULE_ADD:
return 'added rule "%s"' % (self.data['label'], )
elif self.event == AuditLogEntryEvent.RULE_EDIT:
return 'edited rule "%s"' % (self.data['label'], )
elif self.event == AuditLogEntryEvent.RULE_REMOVE:
return 'removed rule "%s"' % (self.data['label'], )
elif self.event == AuditLogEntryEvent.SET_ONDEMAND:
if self.data['ondemand'] == -1:
return 'changed on-demand spend to unlimited'
return 'changed on-demand max spend to $%d' % (self.data['ondemand'] / 100, )
elif self.event == AuditLogEntryEvent.TRIAL_STARTED:
return 'started trial'
elif self.event == AuditLogEntryEvent.PLAN_CHANGED:
return 'changed plan to %s' % (self.data['plan_name'], )
elif self.event == AuditLogEntryEvent.SERVICEHOOK_ADD:
return 'added a service hook for "%s"' % (truncatechars(self.data['url'], 64), )
elif self.event == AuditLogEntryEvent.SERVICEHOOK_EDIT:
return 'edited the service hook for "%s"' % (truncatechars(self.data['url'], 64), )
elif self.event == AuditLogEntryEvent.SERVICEHOOK_REMOVE:
return 'removed the service hook for "%s"' % (truncatechars(self.data['url'], 64), )
elif self.event == AuditLogEntryEvent.SERVICEHOOK_ENABLE:
return 'enabled theservice hook for "%s"' % (truncatechars(self.data['url'], 64), )
elif self.event == AuditLogEntryEvent.SERVICEHOOK_DISABLE:
return 'disabled the service hook for "%s"' % (truncatechars(self.data['url'], 64), )
elif self.event == AuditLogEntryEvent.INTEGRATION_ADD:
return 'enabled integration %s for project %s' % (
self.data['integration'], self.data['project'])
elif self.event == AuditLogEntryEvent.INTEGRATION_EDIT:
return 'edited integration %s for project %s' % (
self.data['integration'], self.data['project'])
elif self.event == AuditLogEntryEvent.INTEGRATION_REMOVE:
return 'disabled integration %s from project %s' % (
self.data['integration'], self.data['project'])
return ''
def validate_data(project, data, client=None):
# TODO(dcramer): move project out of the data packet
data['project'] = project.id
if not data.get('message'):
data['message'] = '<no message value>'
elif not isinstance(data['message'], basestring):
raise APIError('Invalid value for message')
elif len(data['message']) > settings.SENTRY_MAX_MESSAGE_LENGTH:
logger.info(
'Truncated value for message due to length (%d chars)',
len(data['message']), **client_metadata(client, project))
data['message'] = truncatechars(
data['message'], settings.SENTRY_MAX_MESSAGE_LENGTH)
if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH:
logger.info(
'Truncated value for culprit due to length (%d chars)',
len(data['culprit']), **client_metadata(client, project))
data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH)
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
if len(data['event_id']) > 32:
logger.info(
'Discarded value for event_id due to length (%d chars)',
len(data['event_id']), **client_metadata(client, project))
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp as e:
# Log the error, remove the timestamp, and continue
logger.info(
'Discarded invalid value for timestamp: %r', data['timestamp'],
**client_metadata(client, project, exception=e))
del data['timestamp']
if data.get('modules') and type(data['modules']) != dict:
logger.info(
'Discarded invalid type for modules: %s',
type(data['modules']), **client_metadata(client, project))
del data['modules']
if data.get('extra') is not None and type(data['extra']) != dict:
logger.info(
'Discarded invalid type for extra: %s',
type(data['extra']), **client_metadata(client, project))
del data['extra']
if data.get('tags') is not None:
if type(data['tags']) == dict:
data['tags'] = data['tags'].items()
elif not isinstance(data['tags'], (list, tuple)):
logger.info(
'Discarded invalid type for tags: %s',
type(data['tags']), **client_metadata(client, project))
del data['tags']
if data.get('tags'):
# remove any values which are over 32 characters
tags = []
for pair in data['tags']:
try:
k, v = pair
except ValueError:
logger.info('Discarded invalid tag value: %r',
pair, **client_metadata(client, project))
continue
if not isinstance(k, basestring):
try:
k = unicode(k)
except Exception:
logger.info('Discarded invalid tag key: %r',
type(k), **client_metadata(client, project))
continue
if not isinstance(v, basestring):
try:
v = unicode(v)
except Exception:
logger.info('Discarded invalid tag value: %s=%r',
k, type(v), **client_metadata(client, project))
continue
if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH:
logger.info('Discarded invalid tag: %s=%s',
k, v, **client_metadata(client, project))
continue
tags.append((k, v))
data['tags'] = tags
for k in data.keys():
if k in RESERVED_FIELDS:
continue
if not data[k]:
logger.info(
'Ignored empty interface value: %s', k,
**client_metadata(client, project))
del data[k]
continue
import_path = INTERFACE_ALIASES.get(k, k)
if '.' not in import_path:
logger.info(
'Ignored unknown attribute: %s', k,
**client_metadata(client, project))
del data[k]
continue
try:
interface = get_interface(import_path)
except ValueError:
logger.info(
'Invalid unknown attribute: %s', k,
**client_metadata(client, project))
del data[k]
continue
value = data.pop(k)
try:
# HACK: exception allows you to pass the value as a list
# so let's try to actually support that
if isinstance(value, dict):
inst = interface(**value)
else:
inst = interface(value)
inst.validate()
data[import_path] = inst.serialize()
except Exception as e:
if isinstance(e, AssertionError):
log = logger.info
else:
log = logger.error
log('Discarded invalid value for interface: %s', k,
**client_metadata(client, project, exception=e, extra={'value': value}))
level = data.get('level') or DEFAULT_LOG_LEVEL
if isinstance(level, basestring) and not level.isdigit():
# assume it's something like 'warning'
try:
data['level'] = LOG_LEVEL_REVERSE_MAP[level]
except KeyError as e:
logger.info(
'Discarded invalid logger value: %s', level,
**client_metadata(client, project, exception=e))
data['level'] = LOG_LEVEL_REVERSE_MAP.get(
DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL)
return data
def message_top(self):
if self.culprit:
return self.culprit
if not self.message:
return '<unlabeled message>'
return truncatechars(self.message.splitlines()[0], 100)
def fix_culprit(self, data, stacktraces):
culprit_frame = stacktraces[0].frames[-1]
if culprit_frame.module and culprit_frame.function:
data['culprit'] = truncatechars(generate_culprit(culprit_frame), MAX_CULPRIT_LENGTH)
def message_top(self):
if self.culprit:
return self.culprit
return truncatechars(self.message.splitlines()[0], 100)
def expand_javascript_source(data, **kwargs):
"""
Attempt to fetch source code for javascript frames.
Frames must match the following requirements:
- lineno >= 0
- colno >= 0
- abs_path is the HTTP URI to the source
- context_line is empty
Mutates the input ``data`` with expanded context if available.
"""
from sentry.interfaces.stacktrace import Stacktrace
try:
stacktraces = [
Stacktrace.to_python(e['stacktrace'])
for e in data['sentry.interfaces.Exception']['values']
if e.get('stacktrace')
]
except KeyError:
stacktraces = []
if not stacktraces:
logger.debug('No stacktrace for event %r', data['event_id'])
return
# build list of frames that we can actually grab source for
frames = []
for stacktrace in stacktraces:
frames.extend([
f for f in stacktrace.frames
if f.lineno is not None
and f.is_url()
])
if not frames:
logger.debug('Event %r has no frames with enough context to fetch remote source', data['event_id'])
return data
pending_file_list = set()
done_file_list = set()
sourcemap_capable = set()
source_code = {}
sourcemap_idxs = {}
for f in frames:
pending_file_list.add(f.abs_path)
if f.colno is not None:
sourcemap_capable.add(f.abs_path)
while pending_file_list:
filename = pending_file_list.pop()
done_file_list.add(filename)
# TODO: respect cache-contro/max-age headers to some extent
logger.debug('Fetching remote source %r', filename)
result = fetch_url(filename)
if result == BAD_SOURCE:
logger.debug('Bad source file %r', filename)
continue
# If we didn't have a colno, a sourcemap wont do us any good
if filename not in sourcemap_capable:
logger.debug('Not capable of sourcemap: %r', filename)
source_code[filename] = (result.body.splitlines(), None, None)
continue
sourcemap = discover_sourcemap(result)
# TODO: we're currently running splitlines twice
if not sourcemap:
source_code[filename] = (result.body.splitlines(), None, None)
for f in frames:
if not f.module and f.abs_path == filename:
f.module = generate_module(filename)
continue
else:
logger.debug('Found sourcemap %r for minified script %r', sourcemap[:256], result.url)
sourcemap_url = result.url[:1000]
sourcemap_key = hashlib.md5(sourcemap_url).hexdigest()
source_code[filename] = (result.body.splitlines(), sourcemap_url, sourcemap_key)
if sourcemap in sourcemap_idxs:
continue
# pull down sourcemap
index = fetch_sourcemap(sourcemap)
if not index:
logger.debug('Failed parsing sourcemap index: %r', sourcemap[:15])
continue
sourcemap_idxs[sourcemap_key] = (index, sourcemap_url)
# queue up additional source files for download
for source in index.sources:
next_filename = urljoin(sourcemap_url, source)
if next_filename not in done_file_list:
if index.content:
source_code[next_filename] = (index.content[source], None, None)
done_file_list.add(next_filename)
else:
pending_file_list.add(next_filename)
last_state = None
state = None
has_changes = False
for frame in frames:
try:
source, sourcemap_url, sourcemap_key = source_code[frame.abs_path]
except KeyError:
# we must've failed pulling down the source
continue
# may have had a failure pulling down the sourcemap previously
if sourcemap_key in sourcemap_idxs and frame.colno is not None:
index, relative_to = sourcemap_idxs[sourcemap_key]
last_state = state
state = find_source(index, frame.lineno, frame.colno)
abs_path = urljoin(relative_to, state.src)
logger.debug('Mapping compressed source %r to mapping in %r', frame.abs_path, abs_path)
try:
source, _, _ = source_code[abs_path]
except KeyError:
frame.data = {
'sourcemap': sourcemap_url,
}
logger.debug('Failed mapping path %r', abs_path)
else:
# Store original data in annotation
frame.data = {
'orig_lineno': frame.lineno,
'orig_colno': frame.colno,
'orig_function': frame.function,
'orig_abs_path': frame.abs_path,
'orig_filename': frame.filename,
'sourcemap': sourcemap_url,
}
# SourceMap's return zero-indexed lineno's
frame.lineno = state.src_line + 1
frame.colno = state.src_col
# The offending function is always the previous function in the stack
# Honestly, no idea what the bottom most frame is, so we're ignoring that atm
if last_state:
frame.function = last_state.name or frame.function
else:
frame.function = state.name or frame.function
frame.abs_path = abs_path
frame.filename = state.src
frame.module = generate_module(state.src)
elif sourcemap_key in sourcemap_idxs:
frame.data = {
'sourcemap': sourcemap_url,
}
has_changes = True
# TODO: theoretically a minified source could point to another mapped, minified source
frame.pre_context, frame.context_line, frame.post_context = get_source_context(
source=source, lineno=frame.lineno, colno=frame.colno or 0)
if has_changes:
logger.debug('Updating stacktraces with expanded source context')
for exception, stacktrace in itertools.izip(data['sentry.interfaces.Exception']['values'], stacktraces):
exception['stacktrace'] = stacktrace.to_json()
# Attempt to fix the culrpit now that we have potentially useful information
culprit_frame = stacktraces[0].frames[-1]
if culprit_frame.module and culprit_frame.function:
data['culprit'] = truncatechars(generate_culprit(culprit_frame), MAX_CULPRIT_LENGTH)
