def validate_data(project, data, client=None): ensure_valid_project_id(project, data, client=client) if not data.get('message'): data['message'] = '<no message value>' elif not isinstance(data['message'], basestring): raise APIError('Invalid value for message') elif len(data['message']) > MAX_MESSAGE_LENGTH: logger.info('Truncated value for message due to length (%d chars)', len(data['message']), **client_metadata(client, project)) data['message'] = truncatechars(data['message'], MAX_MESSAGE_LENGTH) if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH: logger.info('Truncated value for culprit due to length (%d chars)', len(data['culprit']), **client_metadata(client, project)) data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH) if not data.get('event_id'): data['event_id'] = uuid.uuid4().hex if len(data['event_id']) > 32: logger.info('Discarded value for event_id due to length (%d chars)', len(data['event_id']), **client_metadata(client, project)) data['event_id'] = uuid.uuid4().hex if 'timestamp' in data: try: process_data_timestamp(data) except InvalidTimestamp, e: # Log the error, remove the timestamp, and continue logger.info('Discarded invalid value for timestamp: %r', data['timestamp'], **client_metadata(client, project, exception=e)) del data['timestamp']
def validate_data(project, data, client=None): ensure_valid_project_id(project, data, client=client) if not data.get('message'): data['message'] = '<no message value>' elif not isinstance(data['message'], basestring): raise APIError('Invalid value for message') elif len(data['message']) > MAX_MESSAGE_LENGTH: logger.info('Truncated value for message due to length (%d chars)', len(data['message']), **client_metadata(client)) data['message'] = truncatechars(data['message'], MAX_MESSAGE_LENGTH) if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH: logger.info('Truncated value for culprit due to length (%d chars)', len(data['culprit']), **client_metadata(client)) data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH) if not data.get('event_id'): data['event_id'] = uuid.uuid4().hex if len(data['event_id']) > 32: logger.info('Discarded value for event_id due to length (%d chars)', len(data['event_id']), **client_metadata(client)) data['event_id'] = uuid.uuid4().hex if 'timestamp' in data: try: process_data_timestamp(data) except InvalidTimestamp, e: # Log the error, remove the timestamp, and continue logger.info('Discarded invalid value for timestamp: %r', data['timestamp'], **client_metadata(client, exception=e)) del data['timestamp']
def message_short(self): message = strip(self.message) if not message: message = '<unlabeled message>' else: message = truncatechars(message.splitlines()[0], 100) return message
def generate_culprit(data): platform = data.get('platform') exceptions = get_path(data, 'exception', 'values', filter=True) if exceptions: # Synthetic events no longer get a culprit last_exception = get_path(exceptions, -1) if get_path(last_exception, 'mechanism', 'synthetic'): return '' stacktraces = [e['stacktrace'] for e in exceptions if get_path(e, 'stacktrace', 'frames')] else: stacktrace = data.get('stacktrace') if stacktrace and stacktrace.get('frames'): stacktraces = [stacktrace] else: stacktraces = None culprit = None if not culprit and stacktraces: culprit = get_stacktrace_culprit(get_path(stacktraces, -1), platform=platform) if not culprit and data.get('request'): culprit = get_path(data, 'request', 'url') return truncatechars(culprit or '', MAX_CULPRIT_LENGTH)
def generate_culprit(data): platform = data.get('platform') exceptions = get_path(data, 'exception', 'values', filter=True) if exceptions: # Synthetic events no longer get a culprit last_exception = get_path(exceptions, -1) if get_path(last_exception, 'mechanism', 'synthetic'): return '' stacktraces = [ e['stacktrace'] for e in exceptions if get_path(e, 'stacktrace', 'frames') ] else: stacktrace = data.get('stacktrace') if stacktrace and stacktrace.get('frames'): stacktraces = [stacktrace] else: stacktraces = None culprit = None if not culprit and stacktraces: culprit = get_stacktrace_culprit(get_path(stacktraces, -1), platform=platform) if not culprit and data.get('request'): culprit = get_path(data, 'request', 'url') return truncatechars(culprit or '', MAX_CULPRIT_LENGTH)
def test_truncated(self): url = truncatechars('http://example.com', 3) with pytest.raises(CannotFetchSource) as exc: fetch_file(url) assert exc.value.data['type'] == EventError.JS_MISSING_SOURCE assert exc.value.data['url'] == url
def error(self): message = strip(self.message) if message: message = truncatechars(message, 100) else: message = '<unlabeled message>' return message
def generate_culprit(data, platform=None): culprit = '' try: stacktraces = [ e['stacktrace'] for e in data['sentry.interfaces.Exception']['values'] if e.get('stacktrace') ] except KeyError: stacktrace = data.get('sentry.interfaces.Stacktrace') if stacktrace: stacktraces = [stacktrace] else: stacktraces = None if not stacktraces: if 'sentry.interfaces.Http' in data: culprit = data['sentry.interfaces.Http'].get('url', '') else: from sentry.interfaces.stacktrace import Stacktrace culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string( platform=platform, ) return truncatechars(culprit, MAX_CULPRIT_LENGTH)
def test_truncated(self): url = truncatechars('http://example.com', 3) with pytest.raises(http.CannotFetch) as exc: fetch_file(url) assert exc.value.data['type'] == EventError.JS_MISSING_SOURCE assert exc.value.data['url'] == url
def get_email_subject(self): template = self.project.get_option("mail:subject_template") if template: template = EventSubjectTemplate(template) else: template = DEFAULT_SUBJECT_TEMPLATE return truncatechars(template.safe_substitute(EventSubjectTemplateData(self)), 128)
def to_string(self, metadata): if not metadata['value']: return metadata['type'] return u'{}: {}'.format( metadata['type'], truncatechars(metadata['value'].splitlines()[0], 100), )
def error(self): message = strip(self.get_legacy_message()) if not message: message = '<unlabeled message>' else: message = truncatechars(message.splitlines()[0], 100) return message
def generate_culprit(data): platform = data.get("platform") exceptions = get_path(data, "exception", "values", filter=True) if exceptions: # Synthetic events no longer get a culprit last_exception = get_path(exceptions, -1) if get_path(last_exception, "mechanism", "synthetic"): return "" stacktraces = [ e["stacktrace"] for e in exceptions if get_path(e, "stacktrace", "frames") ] else: stacktrace = data.get("stacktrace") if stacktrace and stacktrace.get("frames"): stacktraces = [stacktrace] else: stacktraces = None culprit = None if not culprit and stacktraces: culprit = get_stacktrace_culprit(get_path(stacktraces, -1), platform=platform) if not culprit and data.get("request"): culprit = get_path(data, "request", "url") return truncatechars(culprit or "", MAX_CULPRIT_LENGTH)
def get_title(self, metadata): ty = metadata.get("type") if ty is None: return metadata.get("function") or "<unknown>" if not metadata.get("value"): return ty return u"{}: {}".format(ty, truncatechars(metadata["value"].splitlines()[0], 100))
def generate_culprit(data, platform=None): exceptions = get_path(data, 'exception', 'values') if exceptions: stacktraces = [ e['stacktrace'] for e in exceptions if get_path(e, 'stacktrace', 'frames') ] else: stacktrace = data.get('stacktrace') if stacktrace and stacktrace.get('frames'): stacktraces = [stacktrace] else: stacktraces = None culprit = None if not culprit and stacktraces: from sentry.interfaces.stacktrace import Stacktrace culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string( platform=platform, ) if not culprit and data.get('request'): culprit = get_path(data, 'request', 'url') return truncatechars(culprit or '', MAX_CULPRIT_LENGTH)
def trim( value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=6, object_hook=None, _depth=0, _size=0, **kwargs, ): """ Truncates a value to ```MAX_VARIABLE_SIZE```. The method of truncation depends on the type of value. """ options = { "max_depth": max_depth, "max_size": max_size, "object_hook": object_hook, "_depth": _depth + 1, } if _depth > max_depth: if not isinstance(value, str): value = json.dumps(value) return trim(value, _size=_size, max_size=max_size) elif isinstance(value, dict): result = {} _size += 2 for k in sorted(value.keys(), key=lambda x: (len(force_text(value[x])), x)): v = value[k] trim_v = trim(v, _size=_size, **options) result[k] = trim_v _size += len(force_text(trim_v)) + 1 if _size >= max_size: break elif isinstance(value, (list, tuple)): result = [] _size += 2 for v in value: trim_v = trim(v, _size=_size, **options) result.append(trim_v) _size += len(force_text(trim_v)) if _size >= max_size: break if isinstance(value, tuple): result = tuple(result) elif isinstance(value, str): result = truncatechars(value, max_size - _size) else: result = value if object_hook is None: return result return object_hook(result)
def message_top(self): culprit = strip(self.culprit) if culprit: return culprit message = strip(self.message) if not strip(message): return '<unlabeled message>' return truncatechars(message.splitlines()[0], 100)
def trim( value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=6, object_hook=None, _depth=0, _size=0, **kwargs ): """ Truncates a value to ```MAX_VARIABLE_SIZE```. The method of truncation depends on the type of value. """ options = { 'max_depth': max_depth, 'max_size': max_size, 'object_hook': object_hook, '_depth': _depth + 1, } if _depth > max_depth and ENABLE_TRIMMING: if not isinstance(value, six.string_types): value = json.dumps(value) return trim(value, _size=_size, max_size=max_size) elif isinstance(value, dict): result = {} _size += 2 for k in sorted(value.keys()): v = value[k] trim_v = trim(v, _size=_size, **options) result[k] = trim_v _size += len(force_text(trim_v)) + 1 if _size >= max_size and ENABLE_TRIMMING: break elif isinstance(value, (list, tuple)): result = [] _size += 2 for v in value: trim_v = trim(v, _size=_size, **options) result.append(trim_v) _size += len(force_text(trim_v)) if _size >= max_size and ENABLE_TRIMMING: break if isinstance(value, tuple): result = tuple(result) elif isinstance(value, six.string_types) and ENABLE_TRIMMING: result = truncatechars(value, max_size - _size) else: result = value if object_hook is None: return result return object_hook(result)
def get_metadata(self): message = strip(self.data.get('message')) if not message: title = '<unlabeled event>' else: title = truncatechars(message.splitlines()[0], 100) return { 'title': title, }
def expose_url(url): if url is None: return u'<unknown>' if url[:5] == 'data:': return u'<data url>' url = truncatechars(url, MAX_URL_LENGTH) if isinstance(url, bytes): url = url.decode('utf-8', 'replace') return url
def expose_url(url): if url is None: return "<unknown>" if url[:5] == "data:": return "<data url>" url = truncatechars(url, MAX_URL_LENGTH) if isinstance(url, six.binary_type): url = url.decode("utf-8", "replace") return url
def expose_url(url): if url is None: return u'<unknown>' if url[:5] == 'data:': return u'<data url>' url = truncatechars(url, MAX_URL_LENGTH) if isinstance(url, six.binary_type): url = url.decode('utf-8', 'replace') return url
def validate_data(project, data, client=None): ensure_valid_project_id(project, data, client=client) if not data.get("message"): data["message"] = "<no message value>" elif not isinstance(data["message"], basestring): raise APIError("Invalid value for message") elif len(data["message"]) > settings.SENTRY_MAX_MESSAGE_LENGTH: logger.info( "Truncated value for message due to length (%d chars)", len(data["message"]), **client_metadata(client, project) ) data["message"] = truncatechars(data["message"], settings.SENTRY_MAX_MESSAGE_LENGTH) if data.get("culprit") and len(data["culprit"]) > MAX_CULPRIT_LENGTH: logger.info( "Truncated value for culprit due to length (%d chars)", len(data["culprit"]), **client_metadata(client, project) ) data["culprit"] = truncatechars(data["culprit"], MAX_CULPRIT_LENGTH) if not data.get("event_id"): data["event_id"] = uuid.uuid4().hex if len(data["event_id"]) > 32: logger.info( "Discarded value for event_id due to length (%d chars)", len(data["event_id"]), **client_metadata(client, project) ) data["event_id"] = uuid.uuid4().hex if "timestamp" in data: try: process_data_timestamp(data) except InvalidTimestamp, e: # Log the error, remove the timestamp, and continue logger.info( "Discarded invalid value for timestamp: %r", data["timestamp"], **client_metadata(client, project, exception=e) ) del data["timestamp"]
def trim( value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=6, object_hook=None, _depth=0, _size=0, **kwargs ): """ Truncates a value to ```MAX_VARIABLE_SIZE```. The method of truncation depends on the type of value. """ options = { 'max_depth': max_depth, 'max_size': max_size, 'object_hook': object_hook, '_depth': _depth + 1, } if _depth > max_depth: if not isinstance(value, six.string_types): value = json.dumps(value) return trim(value, _size=_size, max_size=max_size) elif isinstance(value, dict): result = {} _size += 2 for k in sorted(value.keys()): v = value[k] trim_v = trim(v, _size=_size, **options) result[k] = trim_v _size += len(force_text(trim_v)) + 1 if _size >= max_size: break elif isinstance(value, (list, tuple)): result = [] _size += 2 for v in value: trim_v = trim(v, _size=_size, **options) result.append(trim_v) _size += len(force_text(trim_v)) if _size >= max_size: break elif isinstance(value, six.string_types): result = truncatechars(value, max_size - _size) else: result = value if object_hook is None: return result return object_hook(result)
def fix_culprit(self, data, stacktraces): # This is a bit weird, since the original culprit we get # will be wrong, so we want to touch it up after we've processed # a stack trace. # In this case, we have a list of all stacktraces as a tuple # (stacktrace as dict, stacktrace class) # So we need to take the [1] index to get the Stacktrace class, # then extract the culprit string from that. data["culprit"] = truncatechars(stacktraces[-1][1].get_culprit_string(), MAX_CULPRIT_LENGTH)
def get_title(self, metadata): ty = metadata.get('type') if ty is None: return metadata.get('function') or '<unknown>' if not metadata.get('value'): return ty return u'{}: {}'.format( ty, truncatechars(metadata['value'].splitlines()[0], 100), )
def extract_metadata(self, data): message = strip( get_path(data, "logentry", "formatted") or get_path(data, "logentry", "message")) if message: title = truncatechars(message.splitlines()[0], 100) else: title = "<unlabeled event>" return {"title": title}
def get_email_subject(self): template = self.project.get_option('mail:subject_template') if template: template = EventSubjectTemplate(template) else: template = DEFAULT_SUBJECT_TEMPLATE return truncatechars( template.safe_substitute( EventSubjectTemplateData(self), ), 128, ).encode('utf-8')
def save(self, *args, **kwargs): if not self.last_seen: self.last_seen = timezone.now() if not self.first_seen: self.first_seen = self.last_seen if not self.active_at: self.active_at = self.first_seen # We limit what we store for the message body self.message = strip(self.message) if self.message: self.message = truncatechars(self.message.splitlines()[0], 255) super(Group, self).save(*args, **kwargs)
def get_metadata(self): message = strip(get_path(self.data, 'logentry', 'formatted') or get_path(self.data, 'logentry', 'message')) if message: title = truncatechars(message.splitlines()[0], 100) else: title = '<unlabeled event>' return { 'title': title, }
def get_metadata(self, data): message = strip(get_path(data, 'logentry', 'formatted') or get_path(data, 'logentry', 'message')) if message: title = truncatechars(message.splitlines()[0], 100) else: title = '<unlabeled event>' return { 'title': title, }
def get_metadata(self): # See GH-3248 message_interface = self.data.get('sentry.interfaces.Message', { 'message': self.data.get('message', ''), }) message = strip(message_interface.get('formatted', message_interface['message'])) if not message: title = '<unlabeled event>' else: title = truncatechars(message.splitlines()[0], 100) return { 'title': title, }
def fix_culprit(self, data, stacktraces): # This is a bit weird, since the original culprit we get # will be wrong, so we want to touch it up after we've processed # a stack trace. # In this case, we have a list of all stacktraces as a tuple # (stacktrace as dict, stacktrace class) # So we need to take the [1] index to get the Stacktrace class, # then extract the culprit string from that. data['culprit'] = truncatechars( stacktraces[-1][1].get_culprit_string(), MAX_CULPRIT_LENGTH, )
def trim(value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=3, object_hook=None, _depth=0, _size=0, **kwargs): """ Truncates a value to ```MAX_VARIABLE_SIZE```. The method of truncation depends on the type of value. """ options = { 'max_depth': max_depth, 'max_size': max_size, 'object_hook': object_hook, '_depth': _depth + 1, } if _depth > max_depth: return trim(repr(value), _size=_size, max_size=max_size) elif isinstance(value, dict): result = {} _size += 2 for k, v in value.iteritems(): trim_v = trim(v, _size=_size, **options) result[k] = trim_v _size += len(six.text_type(trim_v)) + 1 if _size >= max_size: break elif isinstance(value, (list, tuple)): result = [] _size += 2 for v in value: trim_v = trim(v, _size=_size, **options) result.append(trim_v) _size += len(six.text_type(trim_v)) if _size >= max_size: break elif isinstance(value, six.string_types): result = truncatechars(value, max_size - _size) else: result = value if object_hook is None: return result return object_hook(result)
def save(self, *args, **kwargs): if not self.last_seen: self.last_seen = timezone.now() if not self.first_seen: self.first_seen = self.last_seen if not self.active_at: self.active_at = self.first_seen # We limit what we store for the message body self.message = strip(self.message) if self.message: self.message = truncatechars(self.message.splitlines()[0], 255) if self.times_seen is None: self.times_seen = 1 self.score = type(self).calculate_score(times_seen=self.times_seen, last_seen=self.last_seen) super().save(*args, **kwargs)
def trim(value, max_size=MAX_VARIABLE_SIZE, max_depth=3, _depth=0, _size=0, **kwargs): """ Truncates a value to ```MAX_VARIABLE_SIZE```. The method of truncation depends on the type of value. """ options = { 'max_depth': max_depth, 'max_size': max_size, '_depth': _depth + 1, } if _depth > max_depth: return trim(repr(value), _size=_size, max_size=max_size) elif isinstance(value, dict): result = {} _size += 2 for k, v in value.iteritems(): trim_v = trim(v, _size=_size, **options) result[k] = trim_v _size += len(unicode(trim_v)) + 1 if _size >= max_size: break elif isinstance(value, (list, tuple)): result = [] _size += 2 for v in value: trim_v = trim(v, _size=_size, **options) result.append(trim_v) _size += len(unicode(trim_v)) if _size >= max_size: break elif isinstance(value, basestring): result = truncatechars(value, max_size - _size) else: result = value return result
def to_string(self, metadata): if metadata.get('message'): return metadata['message'] if metadata.get('title'): return metadata['title'] if metadata.get('type') and metadata.get('value'): return u'{}: {}'.format( metadata['type'], truncatechars(metadata['value'].splitlines()[0], 100), ) if metadata.get('type'): return metadata['type'] return '<unlabeled event>'
def trim(value, max_size=settings.SENTRY_MAX_VARIABLE_SIZE, max_depth=3, _depth=0, _size=0, **kwargs): """ Truncates a value to ```MAX_VARIABLE_SIZE```. The method of truncation depends on the type of value. """ options = { 'max_depth': max_depth, 'max_size': max_size, '_depth': _depth + 1, } if _depth > max_depth: return trim(repr(value), _size=_size, max_size=max_size) elif isinstance(value, dict): result = {} _size += 2 for k, v in value.iteritems(): trim_v = trim(v, _size=_size, **options) result[k] = trim_v _size += len(unicode(trim_v)) + 1 if _size >= max_size: break elif isinstance(value, (list, tuple)): result = [] _size += 2 for v in value: trim_v = trim(v, _size=_size, **options) result.append(trim_v) _size += len(unicode(trim_v)) if _size >= max_size: break elif isinstance(value, basestring): result = truncatechars(value, max_size - _size) else: result = value return result
def generate_culprit(data, platform=None): culprit = "" try: stacktraces = [e["stacktrace"] for e in data["sentry.interfaces.Exception"]["values"] if e.get("stacktrace")] except KeyError: if "sentry.interfaces.Stacktrace" in data: stacktraces = [data["sentry.interfaces.Stacktrace"]] else: stacktraces = None if not stacktraces: if "sentry.interfaces.Http" in data: culprit = data["sentry.interfaces.Http"].get("url", "") else: from sentry.interfaces.stacktrace import Stacktrace culprit = Stacktrace.to_python(stacktraces[-1]).get_culprit_string(platform=platform) return truncatechars(culprit, MAX_CULPRIT_LENGTH)
def test_long_email(self): org = self.create_organization() project = self.create_project(organization=org, name="foo") repo = Repository.objects.create(organization_id=org.id, name="test/repo") release = Release.objects.create(version="abcdabc", organization=org) release.add_project(project) commit_email = "a" * 248 + "@a.com" # 254 chars long, max valid email. release.set_commits([{ "id": "a" * 40, "repository": repo.name, "author_name": "foo bar baz", "author_email": commit_email, "message": "i fixed a bug", }]) commit = Commit.objects.get(repository_id=repo.id, organization_id=org.id, key="a" * 40) assert commit.author.email == truncatechars(commit_email, 75)
def generate_culprit(data): from sentry.interfaces.stacktrace import Stacktrace try: stacktraces = [ e['stacktrace'] for e in data['sentry.interfaces.Exception']['values'] if e.get('stacktrace') ] except KeyError: if 'sentry.interfaces.Stacktrace' in data: stacktraces = [data['sentry.interfaces.Stacktrace']] else: return '' if not stacktraces: return '' return truncatechars( Stacktrace.to_python(stacktraces[-1]).get_culprit_string(), MAX_CULPRIT_LENGTH )
def validate_data(project, data, client=None): # TODO(dcramer): move project out of the data packet data['project'] = project.id if not data.get('message'): data['message'] = '<no message value>' elif not isinstance(data['message'], six.string_types): raise APIError('Invalid value for message') elif len(data['message']) > settings.SENTRY_MAX_MESSAGE_LENGTH: logger.info( 'Truncated value for message due to length (%d chars)', len(data['message']), **client_metadata(client, project)) data['message'] = truncatechars( data['message'], settings.SENTRY_MAX_MESSAGE_LENGTH) if data.get('culprit'): if not isinstance(data['culprit'], six.string_types): raise APIError('Invalid value for culprit') logger.info( 'Truncated value for culprit due to length (%d chars)', len(data['culprit']), **client_metadata(client, project)) data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH) if not data.get('event_id'): data['event_id'] = uuid.uuid4().hex elif not isinstance(data['event_id'], six.string_types): raise APIError('Invalid value for event_id') if len(data['event_id']) > 32: logger.info( 'Discarded value for event_id due to length (%d chars)', len(data['event_id']), **client_metadata(client, project)) data['event_id'] = uuid.uuid4().hex if 'timestamp' in data: try: process_data_timestamp(data) except InvalidTimestamp as e: # Log the error, remove the timestamp, and continue logger.info( 'Discarded invalid value for timestamp: %r', data['timestamp'], **client_metadata(client, project, exception=e)) del data['timestamp'] if data.get('modules') and type(data['modules']) != dict: logger.info( 'Discarded invalid type for modules: %s', type(data['modules']), **client_metadata(client, project)) del data['modules'] if data.get('extra') is not None and type(data['extra']) != dict: logger.info( 'Discarded invalid type for extra: %s', type(data['extra']), **client_metadata(client, project)) del data['extra'] if data.get('tags') is not None: if type(data['tags']) == dict: data['tags'] = data['tags'].items() elif not isinstance(data['tags'], (list, tuple)): logger.info( 'Discarded invalid type for tags: %s', type(data['tags']), **client_metadata(client, project)) del data['tags'] if data.get('tags'): # remove any values which are over 32 characters tags = [] for pair in data['tags']: try: k, v = pair except ValueError: logger.info('Discarded invalid tag value: %r', pair, **client_metadata(client, project)) continue if not isinstance(k, six.string_types): try: k = six.text_type(k) except Exception: logger.info('Discarded invalid tag key: %r', type(k), **client_metadata(client, project)) continue if not isinstance(v, six.string_types): try: v = six.text_type(v) except Exception: logger.info('Discarded invalid tag value: %s=%r', k, type(v), **client_metadata(client, project)) continue if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH: logger.info('Discarded invalid tag: %s=%s', k, v, **client_metadata(client, project)) continue tags.append((k, v)) data['tags'] = tags for k in data.keys(): if k in RESERVED_FIELDS: continue value = data.pop(k) if not value: logger.info( 'Ignored empty interface value: %s', k, **client_metadata(client, project)) continue try: interface = get_interface(k) except ValueError: logger.info( 'Ignored unknown attribute: %s', k, **client_metadata(client, project)) continue if type(value) != dict: # HACK(dcramer): the exception interface supports a list as the # value. We should change this in a new protocol version. if type(value) in (list, tuple): value = {'values': value} else: logger.info( 'Invalid parameters for value: %s', k, type(value), **client_metadata(client, project)) continue try: inst = interface.to_python(value) data[inst.get_path()] = inst.to_json() except Exception as e: if isinstance(e, AssertionError): log = logger.info else: log = logger.error log('Discarded invalid value for interface: %s', k, **client_metadata(client, project, exception=e, extra={'value': value})) level = data.get('level') or DEFAULT_LOG_LEVEL if isinstance(level, six.string_types) and not level.isdigit(): # assume it's something like 'warning' try: data['level'] = LOG_LEVEL_REVERSE_MAP[level] except KeyError as e: logger.info( 'Discarded invalid logger value: %s', level, **client_metadata(client, project, exception=e)) data['level'] = LOG_LEVEL_REVERSE_MAP.get( DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL) return data
def test_truncatechars(): assert truncatechars("12345", 6) == "12345" assert truncatechars("12345", 5) == "12345" assert truncatechars("12345", 4) == "1..." assert truncatechars("12345", 3) == "..." assert truncatechars("12345", 2) == "..." assert truncatechars("12345", 1) == "..." assert truncatechars("12345", 0) == "..." assert truncatechars("12345", 6, ellipsis=u"\u2026") == u"12345" assert truncatechars("12345", 5, ellipsis=u"\u2026") == u"12345" assert truncatechars("12345", 4, ellipsis=u"\u2026") == u"123\u2026" assert truncatechars("12345", 3, ellipsis=u"\u2026") == u"12\u2026" assert truncatechars("12345", 2, ellipsis=u"\u2026") == u"1\u2026" assert truncatechars("12345", 1, ellipsis=u"\u2026") == u"\u2026" assert truncatechars("12345", 0, ellipsis=u"\u2026") == u"\u2026" assert truncatechars(None, 1) is None
def get_note(self): if self.event == AuditLogEntryEvent.MEMBER_INVITE: return 'invited member %s' % (self.data['email'], ) elif self.event == AuditLogEntryEvent.MEMBER_ADD: if self.target_user == self.actor: return 'joined the organization' return 'added member %s' % (self.target_user.get_display_name(), ) elif self.event == AuditLogEntryEvent.MEMBER_ACCEPT: return 'accepted the membership invite' elif self.event == AuditLogEntryEvent.MEMBER_REMOVE: if self.target_user == self.actor: return 'left the organization' return 'removed member %s' % ( self.data.get('email') or self.target_user.get_display_name(), ) elif self.event == AuditLogEntryEvent.MEMBER_EDIT: return 'edited member %s (role: %s, teams: %s)' % ( self.data.get('email') or self.target_user.get_display_name(), self.data.get('role') or 'N/A', ', '.join(six.text_type(x) for x in self.data.get('team_slugs', [])) or 'N/A', ) elif self.event == AuditLogEntryEvent.MEMBER_JOIN_TEAM: if self.target_user == self.actor: return 'joined team %s' % (self.data['team_slug'], ) return 'added %s to team %s' % ( self.data.get('email') or self.target_user.get_display_name(), self.data['team_slug'], ) elif self.event == AuditLogEntryEvent.MEMBER_LEAVE_TEAM: if self.target_user == self.actor: return 'left team %s' % (self.data['team_slug'], ) return 'removed %s from team %s' % ( self.data.get('email') or self.target_user.get_display_name(), self.data['team_slug'], ) elif self.event == AuditLogEntryEvent.MEMBER_PENDING: return 'required member %s to setup 2FA' % ( self.data.get('email') or self.target_user.get_display_name(), ) elif self.event == AuditLogEntryEvent.ORG_ADD: return 'created the organization' elif self.event == AuditLogEntryEvent.ORG_EDIT: return 'edited the organization setting: ' + (', '.join(u'{} {}'.format(k, v) for k, v in self.data.items())) elif self.event == AuditLogEntryEvent.ORG_REMOVE: return 'removed the organization' elif self.event == AuditLogEntryEvent.ORG_RESTORE: return 'restored the organization' elif self.event == AuditLogEntryEvent.TEAM_ADD: return 'created team %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.TEAM_EDIT: return 'edited team %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.TEAM_REMOVE: return 'removed team %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.PROJECT_ADD: return 'created project %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.PROJECT_EDIT: return 'edited project settings ' + (' '.join([' in %s to %s' % (key, value) for (key, value) in six.iteritems(self.data)])) elif self.event == AuditLogEntryEvent.PROJECT_REMOVE: return 'removed project %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.PROJECT_REQUEST_TRANSFER: return 'requested to transfer project %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.PROJECT_ACCEPT_TRANSFER: return 'accepted transfer of project %s' % (self.data['slug'], ) elif self.event == AuditLogEntryEvent.TAGKEY_REMOVE: return 'removed tags matching %s = *' % (self.data['key'], ) elif self.event == AuditLogEntryEvent.PROJECTKEY_ADD: return 'added project key %s' % (self.data['public_key'], ) elif self.event == AuditLogEntryEvent.PROJECTKEY_EDIT: return 'edited project key %s' % (self.data['public_key'], ) elif self.event == AuditLogEntryEvent.PROJECTKEY_REMOVE: return 'removed project key %s' % (self.data['public_key'], ) elif self.event == AuditLogEntryEvent.PROJECTKEY_ENABLE: return 'enabled project key %s' % (self.data['public_key'], ) elif self.event == AuditLogEntryEvent.PROJECTKEY_DISABLE: return 'disabled project key %s' % (self.data['public_key'], ) elif self.event == AuditLogEntryEvent.SSO_ENABLE: return 'enabled sso (%s)' % (self.data['provider'], ) elif self.event == AuditLogEntryEvent.SSO_DISABLE: return 'disabled sso (%s)' % (self.data['provider'], ) elif self.event == AuditLogEntryEvent.SSO_EDIT: return 'edited sso settings' elif self.event == AuditLogEntryEvent.SSO_IDENTITY_LINK: return 'linked their account to a new identity' elif self.event == AuditLogEntryEvent.APIKEY_ADD: return 'added api key %s' % (self.data['label'], ) elif self.event == AuditLogEntryEvent.APIKEY_EDIT: return 'edited api key %s' % (self.data['label'], ) elif self.event == AuditLogEntryEvent.APIKEY_REMOVE: return 'removed api key %s' % (self.data['label'], ) elif self.event == AuditLogEntryEvent.RULE_ADD: return 'added rule "%s"' % (self.data['label'], ) elif self.event == AuditLogEntryEvent.RULE_EDIT: return 'edited rule "%s"' % (self.data['label'], ) elif self.event == AuditLogEntryEvent.RULE_REMOVE: return 'removed rule "%s"' % (self.data['label'], ) elif self.event == AuditLogEntryEvent.SET_ONDEMAND: if self.data['ondemand'] == -1: return 'changed on-demand spend to unlimited' return 'changed on-demand max spend to $%d' % (self.data['ondemand'] / 100, ) elif self.event == AuditLogEntryEvent.TRIAL_STARTED: return 'started trial' elif self.event == AuditLogEntryEvent.PLAN_CHANGED: return 'changed plan to %s' % (self.data['plan_name'], ) elif self.event == AuditLogEntryEvent.SERVICEHOOK_ADD: return 'added a service hook for "%s"' % (truncatechars(self.data['url'], 64), ) elif self.event == AuditLogEntryEvent.SERVICEHOOK_EDIT: return 'edited the service hook for "%s"' % (truncatechars(self.data['url'], 64), ) elif self.event == AuditLogEntryEvent.SERVICEHOOK_REMOVE: return 'removed the service hook for "%s"' % (truncatechars(self.data['url'], 64), ) elif self.event == AuditLogEntryEvent.SERVICEHOOK_ENABLE: return 'enabled theservice hook for "%s"' % (truncatechars(self.data['url'], 64), ) elif self.event == AuditLogEntryEvent.SERVICEHOOK_DISABLE: return 'disabled the service hook for "%s"' % (truncatechars(self.data['url'], 64), ) elif self.event == AuditLogEntryEvent.INTEGRATION_ADD: return 'enabled integration %s for project %s' % ( self.data['integration'], self.data['project']) elif self.event == AuditLogEntryEvent.INTEGRATION_EDIT: return 'edited integration %s for project %s' % ( self.data['integration'], self.data['project']) elif self.event == AuditLogEntryEvent.INTEGRATION_REMOVE: return 'disabled integration %s from project %s' % ( self.data['integration'], self.data['project']) return ''
def validate_data(project, data, client=None): # TODO(dcramer): move project out of the data packet data['project'] = project.id if not data.get('message'): data['message'] = '<no message value>' elif not isinstance(data['message'], basestring): raise APIError('Invalid value for message') elif len(data['message']) > settings.SENTRY_MAX_MESSAGE_LENGTH: logger.info( 'Truncated value for message due to length (%d chars)', len(data['message']), **client_metadata(client, project)) data['message'] = truncatechars( data['message'], settings.SENTRY_MAX_MESSAGE_LENGTH) if data.get('culprit') and len(data['culprit']) > MAX_CULPRIT_LENGTH: logger.info( 'Truncated value for culprit due to length (%d chars)', len(data['culprit']), **client_metadata(client, project)) data['culprit'] = truncatechars(data['culprit'], MAX_CULPRIT_LENGTH) if not data.get('event_id'): data['event_id'] = uuid.uuid4().hex if len(data['event_id']) > 32: logger.info( 'Discarded value for event_id due to length (%d chars)', len(data['event_id']), **client_metadata(client, project)) data['event_id'] = uuid.uuid4().hex if 'timestamp' in data: try: process_data_timestamp(data) except InvalidTimestamp as e: # Log the error, remove the timestamp, and continue logger.info( 'Discarded invalid value for timestamp: %r', data['timestamp'], **client_metadata(client, project, exception=e)) del data['timestamp'] if data.get('modules') and type(data['modules']) != dict: logger.info( 'Discarded invalid type for modules: %s', type(data['modules']), **client_metadata(client, project)) del data['modules'] if data.get('extra') is not None and type(data['extra']) != dict: logger.info( 'Discarded invalid type for extra: %s', type(data['extra']), **client_metadata(client, project)) del data['extra'] if data.get('tags') is not None: if type(data['tags']) == dict: data['tags'] = data['tags'].items() elif not isinstance(data['tags'], (list, tuple)): logger.info( 'Discarded invalid type for tags: %s', type(data['tags']), **client_metadata(client, project)) del data['tags'] if data.get('tags'): # remove any values which are over 32 characters tags = [] for pair in data['tags']: try: k, v = pair except ValueError: logger.info('Discarded invalid tag value: %r', pair, **client_metadata(client, project)) continue if not isinstance(k, basestring): try: k = unicode(k) except Exception: logger.info('Discarded invalid tag key: %r', type(k), **client_metadata(client, project)) continue if not isinstance(v, basestring): try: v = unicode(v) except Exception: logger.info('Discarded invalid tag value: %s=%r', k, type(v), **client_metadata(client, project)) continue if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH: logger.info('Discarded invalid tag: %s=%s', k, v, **client_metadata(client, project)) continue tags.append((k, v)) data['tags'] = tags for k in data.keys(): if k in RESERVED_FIELDS: continue if not data[k]: logger.info( 'Ignored empty interface value: %s', k, **client_metadata(client, project)) del data[k] continue import_path = INTERFACE_ALIASES.get(k, k) if '.' not in import_path: logger.info( 'Ignored unknown attribute: %s', k, **client_metadata(client, project)) del data[k] continue try: interface = get_interface(import_path) except ValueError: logger.info( 'Invalid unknown attribute: %s', k, **client_metadata(client, project)) del data[k] continue value = data.pop(k) try: # HACK: exception allows you to pass the value as a list # so let's try to actually support that if isinstance(value, dict): inst = interface(**value) else: inst = interface(value) inst.validate() data[import_path] = inst.serialize() except Exception as e: if isinstance(e, AssertionError): log = logger.info else: log = logger.error log('Discarded invalid value for interface: %s', k, **client_metadata(client, project, exception=e, extra={'value': value})) level = data.get('level') or DEFAULT_LOG_LEVEL if isinstance(level, basestring) and not level.isdigit(): # assume it's something like 'warning' try: data['level'] = LOG_LEVEL_REVERSE_MAP[level] except KeyError as e: logger.info( 'Discarded invalid logger value: %s', level, **client_metadata(client, project, exception=e)) data['level'] = LOG_LEVEL_REVERSE_MAP.get( DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL) return data
def message_top(self): if self.culprit: return self.culprit if not self.message: return '<unlabeled message>' return truncatechars(self.message.splitlines()[0], 100)
def fix_culprit(self, data, stacktraces): culprit_frame = stacktraces[0].frames[-1] if culprit_frame.module and culprit_frame.function: data['culprit'] = truncatechars(generate_culprit(culprit_frame), MAX_CULPRIT_LENGTH)
def message_top(self): if self.culprit: return self.culprit return truncatechars(self.message.splitlines()[0], 100)
def expand_javascript_source(data, **kwargs): """ Attempt to fetch source code for javascript frames. Frames must match the following requirements: - lineno >= 0 - colno >= 0 - abs_path is the HTTP URI to the source - context_line is empty Mutates the input ``data`` with expanded context if available. """ from sentry.interfaces.stacktrace import Stacktrace try: stacktraces = [ Stacktrace.to_python(e['stacktrace']) for e in data['sentry.interfaces.Exception']['values'] if e.get('stacktrace') ] except KeyError: stacktraces = [] if not stacktraces: logger.debug('No stacktrace for event %r', data['event_id']) return # build list of frames that we can actually grab source for frames = [] for stacktrace in stacktraces: frames.extend([ f for f in stacktrace.frames if f.lineno is not None and f.is_url() ]) if not frames: logger.debug('Event %r has no frames with enough context to fetch remote source', data['event_id']) return data pending_file_list = set() done_file_list = set() sourcemap_capable = set() source_code = {} sourcemap_idxs = {} for f in frames: pending_file_list.add(f.abs_path) if f.colno is not None: sourcemap_capable.add(f.abs_path) while pending_file_list: filename = pending_file_list.pop() done_file_list.add(filename) # TODO: respect cache-contro/max-age headers to some extent logger.debug('Fetching remote source %r', filename) result = fetch_url(filename) if result == BAD_SOURCE: logger.debug('Bad source file %r', filename) continue # If we didn't have a colno, a sourcemap wont do us any good if filename not in sourcemap_capable: logger.debug('Not capable of sourcemap: %r', filename) source_code[filename] = (result.body.splitlines(), None, None) continue sourcemap = discover_sourcemap(result) # TODO: we're currently running splitlines twice if not sourcemap: source_code[filename] = (result.body.splitlines(), None, None) for f in frames: if not f.module and f.abs_path == filename: f.module = generate_module(filename) continue else: logger.debug('Found sourcemap %r for minified script %r', sourcemap[:256], result.url) sourcemap_url = result.url[:1000] sourcemap_key = hashlib.md5(sourcemap_url).hexdigest() source_code[filename] = (result.body.splitlines(), sourcemap_url, sourcemap_key) if sourcemap in sourcemap_idxs: continue # pull down sourcemap index = fetch_sourcemap(sourcemap) if not index: logger.debug('Failed parsing sourcemap index: %r', sourcemap[:15]) continue sourcemap_idxs[sourcemap_key] = (index, sourcemap_url) # queue up additional source files for download for source in index.sources: next_filename = urljoin(sourcemap_url, source) if next_filename not in done_file_list: if index.content: source_code[next_filename] = (index.content[source], None, None) done_file_list.add(next_filename) else: pending_file_list.add(next_filename) last_state = None state = None has_changes = False for frame in frames: try: source, sourcemap_url, sourcemap_key = source_code[frame.abs_path] except KeyError: # we must've failed pulling down the source continue # may have had a failure pulling down the sourcemap previously if sourcemap_key in sourcemap_idxs and frame.colno is not None: index, relative_to = sourcemap_idxs[sourcemap_key] last_state = state state = find_source(index, frame.lineno, frame.colno) abs_path = urljoin(relative_to, state.src) logger.debug('Mapping compressed source %r to mapping in %r', frame.abs_path, abs_path) try: source, _, _ = source_code[abs_path] except KeyError: frame.data = { 'sourcemap': sourcemap_url, } logger.debug('Failed mapping path %r', abs_path) else: # Store original data in annotation frame.data = { 'orig_lineno': frame.lineno, 'orig_colno': frame.colno, 'orig_function': frame.function, 'orig_abs_path': frame.abs_path, 'orig_filename': frame.filename, 'sourcemap': sourcemap_url, } # SourceMap's return zero-indexed lineno's frame.lineno = state.src_line + 1 frame.colno = state.src_col # The offending function is always the previous function in the stack # Honestly, no idea what the bottom most frame is, so we're ignoring that atm if last_state: frame.function = last_state.name or frame.function else: frame.function = state.name or frame.function frame.abs_path = abs_path frame.filename = state.src frame.module = generate_module(state.src) elif sourcemap_key in sourcemap_idxs: frame.data = { 'sourcemap': sourcemap_url, } has_changes = True # TODO: theoretically a minified source could point to another mapped, minified source frame.pre_context, frame.context_line, frame.post_context = get_source_context( source=source, lineno=frame.lineno, colno=frame.colno or 0) if has_changes: logger.debug('Updating stacktraces with expanded source context') for exception, stacktrace in itertools.izip(data['sentry.interfaces.Exception']['values'], stacktraces): exception['stacktrace'] = stacktrace.to_json() # Attempt to fix the culrpit now that we have potentially useful information culprit_frame = stacktraces[0].frames[-1] if culprit_frame.module and culprit_frame.function: data['culprit'] = truncatechars(generate_culprit(culprit_frame), MAX_CULPRIT_LENGTH)