def normalize(self):
# TODO(dcramer): store http.env.REMOTE_ADDR as user.ip
# First we pull out our top-level (non-data attr) kwargs
data = self.data
if not isinstance(data.get('level'), (six.string_types, int)):
data['level'] = logging.ERROR
elif data['level'] not in LOG_LEVELS:
data['level'] = logging.ERROR
if not data.get('logger'):
data['logger'] = DEFAULT_LOGGER_NAME
else:
logger = trim(data['logger'].strip(), 64)
if TagKey.is_valid_key(logger):
data['logger'] = logger
else:
data['logger'] = DEFAULT_LOGGER_NAME
if data.get('platform'):
data['platform'] = trim(data['platform'], 64)
current_timestamp = timezone.now()
timestamp = data.get('timestamp')
if not timestamp:
timestamp = current_timestamp
if isinstance(timestamp, datetime):
# We must convert date to local time so Django doesn't mess it up
# based on TIME_ZONE
if settings.TIME_ZONE:
if not timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=timezone.utc)
elif timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=None)
timestamp = float(timestamp.strftime('%s'))
data['timestamp'] = timestamp
data['received'] = float(timezone.now().strftime('%s'))
if not data.get('event_id'):
data['event_id'] = uuid4().hex
data.setdefault('culprit', None)
data.setdefault('server_name', None)
data.setdefault('site', None)
data.setdefault('checksum', None)
data.setdefault('fingerprint', None)
data.setdefault('platform', None)
data.setdefault('environment', None)
data.setdefault('extra', {})
data.setdefault('errors', [])
tags = data.get('tags')
if not tags:
tags = []
# full support for dict syntax
elif isinstance(tags, dict):
tags = list(tags.items())
# prevent [tag, tag, tag] (invalid) syntax
elif not all(len(t) == 2 for t in tags):
tags = []
else:
tags = list(tags)
data['tags'] = []
for key, value in tags:
key = six.text_type(key).strip()
value = six.text_type(value).strip()
if not (key and value):
continue
# XXX(dcramer): many legacy apps are using the environment tag
# rather than the key itself
if key == 'environment' and not data.get('environment'):
data['environment'] = value
else:
data['tags'].append((key, value))
if not isinstance(data['extra'], dict):
# throw it away
data['extra'] = {}
trim_dict(
data['extra'], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE)
# TODO(dcramer): more of validate data needs stuffed into the manager
for key in list(iter(data)):
if key in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(key)
try:
interface = get_interface(key)()
except ValueError:
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception:
# XXX: we should consider logging this.
pass
# TODO(dcramer): this logic is duplicated in ``validate_data`` from
# coreapi
# message is coerced to an interface, as its used for pure
# index of searchable strings
# See GH-3248
message = data.pop('message', None)
if message:
if 'sentry.interfaces.Message' not in data:
interface = get_interface('sentry.interfaces.Message')
try:
inst = interface.to_python({
'message': message,
})
data[inst.get_path()] = inst.to_json()
except Exception:
pass
elif not data['sentry.interfaces.Message'].get('formatted'):
interface = get_interface('sentry.interfaces.Message')
try:
inst = interface.to_python(dict(
data['sentry.interfaces.Message'],
formatted=message,
))
data[inst.get_path()] = inst.to_json()
except Exception:
pass
# the SDKs currently do not describe event types, and we must infer
# them from available attributes
data['type'] = eventtypes.infer(data).key
data['version'] = self.version
# TODO(dcramer): find a better place for this logic
exception = data.get('sentry.interfaces.Exception')
stacktrace = data.get('sentry.interfaces.Stacktrace')
if exception and len(exception['values']) == 1 and stacktrace:
exception['values'][0]['stacktrace'] = stacktrace
del data['sentry.interfaces.Stacktrace']
if 'sentry.interfaces.Http' in data:
try:
ip_address = validate_ip(
data['sentry.interfaces.Http'].get(
'env', {}).get('REMOTE_ADDR'),
required=False,
)
except ValueError:
ip_address = None
if ip_address:
data.setdefault('sentry.interfaces.User', {})
data['sentry.interfaces.User'].setdefault(
'ip_address', ip_address)
if data['culprit']:
data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH)
return data
def normalize(self):
# TODO(dcramer): store http.env.REMOTE_ADDR as user.ip
# First we pull out our top-level (non-data attr) kwargs
data = self.data
if not isinstance(data.get('level'), (six.string_types, int)):
data['level'] = logging.ERROR
elif data['level'] not in LOG_LEVELS:
data['level'] = logging.ERROR
if not data.get('logger'):
data['logger'] = DEFAULT_LOGGER_NAME
else:
logger = trim(data['logger'].strip(), 64)
if TagKey.is_valid_key(logger):
data['logger'] = logger
else:
data['logger'] = DEFAULT_LOGGER_NAME
if data.get('platform'):
data['platform'] = trim(data['platform'], 64)
timestamp = data.get('timestamp')
if not timestamp:
timestamp = timezone.now()
if isinstance(timestamp, datetime):
# We must convert date to local time so Django doesn't mess it up
# based on TIME_ZONE
if settings.TIME_ZONE:
if not timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=timezone.utc)
elif timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=None)
timestamp = float(timestamp.strftime('%s'))
data['timestamp'] = timestamp
if not data.get('event_id'):
data['event_id'] = uuid4().hex
data.setdefault('message', '')
data.setdefault('culprit', None)
data.setdefault('time_spent', None)
data.setdefault('server_name', None)
data.setdefault('site', None)
data.setdefault('checksum', None)
data.setdefault('fingerprint', None)
data.setdefault('platform', None)
data.setdefault('environment', None)
data.setdefault('extra', {})
data.setdefault('errors', [])
tags = data.get('tags')
if not tags:
tags = []
# full support for dict syntax
elif isinstance(tags, dict):
tags = tags.items()
# prevent [tag, tag, tag] (invalid) syntax
elif not all(len(t) == 2 for t in tags):
tags = []
else:
tags = list(tags)
data['tags'] = []
for key, value in tags:
key = six.text_type(key).strip()
value = six.text_type(value).strip()
if not (key and value):
continue
data['tags'].append((key, value))
if not isinstance(data['extra'], dict):
# throw it away
data['extra'] = {}
trim_dict(
data['extra'], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE)
# TODO(dcramer): more of validate data needs stuffed into the manager
for key in data.keys():
if key in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(key)
try:
interface = get_interface(key)()
except ValueError:
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception:
pass
data['version'] = self.version
# TODO(dcramer): find a better place for this logic
exception = data.get('sentry.interfaces.Exception')
stacktrace = data.get('sentry.interfaces.Stacktrace')
if exception and len(exception['values']) == 1 and stacktrace:
exception['values'][0]['stacktrace'] = stacktrace
del data['sentry.interfaces.Stacktrace']
if 'sentry.interfaces.Http' in data:
try:
ip_address = validate_ip(
data['sentry.interfaces.Http'].get(
'env', {}).get('REMOTE_ADDR'),
required=False,
)
except ValueError:
ip_address = None
if ip_address:
data.setdefault('sentry.interfaces.User', {})
data['sentry.interfaces.User'].setdefault(
'ip_address', ip_address)
if data['time_spent']:
data['time_spent'] = int(data['time_spent'])
if data['culprit']:
data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH)
if data['message']:
data['message'] = trim(
data['message'], settings.SENTRY_MAX_MESSAGE_LENGTH)
return data
def validate_data(self, project, data):
# TODO(dcramer): move project out of the data packet
data['project'] = project.id
data['errors'] = []
if data.get('culprit'):
if not isinstance(data['culprit'], six.string_types):
raise APIForbidden('Invalid value for culprit')
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif not isinstance(data['event_id'], six.string_types):
raise APIForbidden('Invalid value for event_id')
if len(data['event_id']) > 32:
self.log.debug(
'Discarded value for event_id due to length (%d chars)',
len(data['event_id']))
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'event_id',
'value': data['event_id'],
})
data['event_id'] = uuid.uuid4().hex
elif not is_event_id(data['event_id']):
self.log.debug('Discarded invalid value for event_id: %r',
data['event_id'],
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'event_id',
'value': data['event_id'],
})
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
self._process_data_timestamp(data)
except InvalidTimestamp as e:
self.log.debug('Discarded invalid value for timestamp: %r',
data['timestamp'],
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'timestamp',
'value': data['timestamp'],
})
del data['timestamp']
if 'fingerprint' in data:
try:
self._process_fingerprint(data)
except InvalidFingerprint as e:
self.log.debug('Discarded invalid value for fingerprint: %r',
data['fingerprint'],
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'fingerprint',
'value': data['fingerprint'],
})
del data['fingerprint']
if 'platform' not in data or data['platform'] not in VALID_PLATFORMS:
data['platform'] = 'other'
if data.get('modules') and type(data['modules']) != dict:
self.log.debug('Discarded invalid type for modules: %s',
type(data['modules']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'modules',
'value': data['modules'],
})
del data['modules']
if data.get('extra') is not None and type(data['extra']) != dict:
self.log.debug('Discarded invalid type for extra: %s',
type(data['extra']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'extra',
'value': data['extra'],
})
del data['extra']
if data.get('tags') is not None:
if type(data['tags']) == dict:
data['tags'] = list(data['tags'].items())
elif not isinstance(data['tags'], (list, tuple)):
self.log.debug('Discarded invalid type for tags: %s',
type(data['tags']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': data['tags'],
})
del data['tags']
if data.get('tags'):
# remove any values which are over 32 characters
tags = []
for pair in data['tags']:
try:
k, v = pair
except ValueError:
self.log.debug('Discarded invalid tag value: %r', pair)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not isinstance(k, six.string_types):
try:
k = six.text_type(k)
except Exception:
self.log.debug('Discarded invalid tag key: %r',
type(k))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not isinstance(v, six.string_types):
try:
v = six.text_type(v)
except Exception:
self.log.debug('Discarded invalid tag value: %s=%r', k,
type(v))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if len(k) > MAX_TAG_KEY_LENGTH or len(
v) > MAX_TAG_VALUE_LENGTH:
self.log.debug('Discarded invalid tag: %s=%s', k, v)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
# support tags with spaces by converting them
k = k.replace(' ', '-')
if TagKey.is_reserved_key(k):
self.log.debug('Discarding reserved tag key: %s', k)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not TagKey.is_valid_key(k):
self.log.debug('Discarded invalid tag key: %s', k)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not TagValue.is_valid_value(v):
self.log.debug('Discard invalid tag value: %s', v)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
tags.append((k, v))
data['tags'] = tags
for k in list(iter(data)):
if k in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(k)
if not value:
self.log.debug('Ignored empty interface value: %s', k)
continue
try:
interface = get_interface(k)
except ValueError:
self.log.debug('Ignored unknown attribute: %s', k)
data['errors'].append({
'type': EventError.INVALID_ATTRIBUTE,
'name': k,
})
continue
if type(value) != dict:
# HACK(dcramer): the exception/breadcrumbs interface supports a
# list as the value. We should change this in a new protocol
# version.
if type(value) in (list, tuple):
value = {'values': value}
else:
self.log.debug('Invalid parameter for value: %s (%r)', k,
type(value))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, InterfaceValidationError):
log = self.log.debug
else:
log = self.log.error
log('Discarded invalid value for interface: %s (%r)',
k,
value,
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
# TODO(dcramer): ideally this logic would happen in normalize, but today
# we don't do "validation" there (create errors)
# message is coerced to an interface, as its used for pure
# index of searchable strings
# See GH-3248
message = data.pop('message', None)
if message:
if 'sentry.interfaces.Message' not in data:
value = {
'message': message,
}
elif not data['sentry.interfaces.Message'].get('formatted'):
value = data['sentry.interfaces.Message']
value['formatted'] = message
else:
value = None
if value is not None:
k = 'sentry.interfaces.Message'
interface = get_interface(k)
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, InterfaceValidationError):
log = self.log.debug
else:
log = self.log.error
log('Discarded invalid value for interface: %s (%r)',
k,
value,
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
level = data.get('level') or DEFAULT_LOG_LEVEL
if isinstance(level, six.string_types) and not level.isdigit():
# assume it's something like 'warning'
try:
data['level'] = LOG_LEVELS_MAP[level]
except KeyError as e:
self.log.debug('Discarded invalid logger value: %s', level)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'level',
'value': level,
})
data['level'] = LOG_LEVELS_MAP.get(DEFAULT_LOG_LEVEL,
DEFAULT_LOG_LEVEL)
if data.get('release'):
data['release'] = six.text_type(data['release'])
if len(data['release']) > 64:
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'release',
'value': data['release'],
})
del data['release']
return data
def normalize(self):
# TODO(dcramer): store http.env.REMOTE_ADDR as user.ip
# First we pull out our top-level (non-data attr) kwargs
data = self.data
if not isinstance(data.get('level'), (six.string_types, int)):
data['level'] = logging.ERROR
elif data['level'] not in LOG_LEVELS:
data['level'] = logging.ERROR
if not data.get('logger'):
data['logger'] = DEFAULT_LOGGER_NAME
else:
logger = trim(data['logger'].strip(), 64)
if TagKey.is_valid_key(logger):
data['logger'] = logger
else:
data['logger'] = DEFAULT_LOGGER_NAME
if data.get('platform'):
data['platform'] = trim(data['platform'], 64)
timestamp = data.get('timestamp')
if not timestamp:
timestamp = timezone.now()
if isinstance(timestamp, datetime):
# We must convert date to local time so Django doesn't mess it up
# based on TIME_ZONE
if settings.TIME_ZONE:
if not timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=timezone.utc)
elif timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=None)
timestamp = float(timestamp.strftime('%s'))
data['timestamp'] = timestamp
if not data.get('event_id'):
data['event_id'] = uuid4().hex
data.setdefault('message', '')
data.setdefault('culprit', None)
data.setdefault('time_spent', None)
data.setdefault('server_name', None)
data.setdefault('site', None)
data.setdefault('checksum', None)
data.setdefault('fingerprint', None)
data.setdefault('platform', None)
data.setdefault('environment', None)
data.setdefault('extra', {})
data.setdefault('errors', [])
tags = data.get('tags')
if not tags:
tags = []
# full support for dict syntax
elif isinstance(tags, dict):
tags = tags.items()
# prevent [tag, tag, tag] (invalid) syntax
elif not all(len(t) == 2 for t in tags):
tags = []
else:
tags = list(tags)
data['tags'] = []
for key, value in tags:
key = six.text_type(key).strip()
value = six.text_type(value).strip()
if not (key and value):
continue
data['tags'].append((key, value))
if not isinstance(data['extra'], dict):
# throw it away
data['extra'] = {}
trim_dict(data['extra'],
max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE)
# TODO(dcramer): more of validate data needs stuffed into the manager
for key in data.keys():
if key in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(key)
try:
interface = get_interface(key)()
except ValueError:
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception:
pass
data['version'] = self.version
# TODO(dcramer): find a better place for this logic
exception = data.get('sentry.interfaces.Exception')
stacktrace = data.get('sentry.interfaces.Stacktrace')
if exception and len(exception['values']) == 1 and stacktrace:
exception['values'][0]['stacktrace'] = stacktrace
del data['sentry.interfaces.Stacktrace']
if 'sentry.interfaces.Http' in data:
try:
ip_address = validate_ip(
data['sentry.interfaces.Http'].get('env',
{}).get('REMOTE_ADDR'),
required=False,
)
except ValueError:
ip_address = None
if ip_address:
data.setdefault('sentry.interfaces.User', {})
data['sentry.interfaces.User'].setdefault(
'ip_address', ip_address)
if data['time_spent']:
data['time_spent'] = int(data['time_spent'])
if data['culprit']:
data['culprit'] = trim(data['culprit'], MAX_CULPRIT_LENGTH)
if data['message']:
data['message'] = trim(data['message'],
settings.SENTRY_MAX_MESSAGE_LENGTH)
return data
def validate_data(self, project, data):
# TODO(dcramer): move project out of the data packet
data['project'] = project.id
data['errors'] = []
if data.get('culprit'):
if not isinstance(data['culprit'], six.string_types):
raise APIForbidden('Invalid value for culprit')
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif not isinstance(data['event_id'], six.string_types):
raise APIForbidden('Invalid value for event_id')
if len(data['event_id']) > 32:
self.log.debug(
'Discarded value for event_id due to length (%d chars)',
len(data['event_id']))
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'event_id',
'value': data['event_id'],
})
data['event_id'] = uuid.uuid4().hex
elif not is_event_id(data['event_id']):
self.log.debug(
'Discarded invalid value for event_id: %r',
data['event_id'], exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'event_id',
'value': data['event_id'],
})
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
self._process_data_timestamp(data)
except InvalidTimestamp as e:
self.log.debug(
'Discarded invalid value for timestamp: %r',
data['timestamp'], exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'timestamp',
'value': data['timestamp'],
})
del data['timestamp']
if 'fingerprint' in data:
try:
self._process_fingerprint(data)
except InvalidFingerprint as e:
self.log.debug(
'Discarded invalid value for fingerprint: %r',
data['fingerprint'], exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'fingerprint',
'value': data['fingerprint'],
})
del data['fingerprint']
if 'platform' not in data or data['platform'] not in VALID_PLATFORMS:
data['platform'] = 'other'
if data.get('modules') and type(data['modules']) != dict:
self.log.debug(
'Discarded invalid type for modules: %s',
type(data['modules']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'modules',
'value': data['modules'],
})
del data['modules']
if data.get('extra') is not None and type(data['extra']) != dict:
self.log.debug(
'Discarded invalid type for extra: %s',
type(data['extra']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'extra',
'value': data['extra'],
})
del data['extra']
if data.get('tags') is not None:
if type(data['tags']) == dict:
data['tags'] = list(data['tags'].items())
elif not isinstance(data['tags'], (list, tuple)):
self.log.debug(
'Discarded invalid type for tags: %s', type(data['tags']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': data['tags'],
})
del data['tags']
if data.get('tags'):
# remove any values which are over 32 characters
tags = []
for pair in data['tags']:
try:
k, v = pair
except ValueError:
self.log.debug('Discarded invalid tag value: %r', pair)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not isinstance(k, six.string_types):
try:
k = six.text_type(k)
except Exception:
self.log.debug('Discarded invalid tag key: %r', type(k))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not isinstance(v, six.string_types):
try:
v = six.text_type(v)
except Exception:
self.log.debug('Discarded invalid tag value: %s=%r',
k, type(v))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH:
self.log.debug('Discarded invalid tag: %s=%s', k, v)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
# support tags with spaces by converting them
k = k.replace(' ', '-')
if TagKey.is_reserved_key(k):
self.log.debug('Discarding reserved tag key: %s', k)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not TagKey.is_valid_key(k):
self.log.debug('Discarded invalid tag key: %s', k)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not TagValue.is_valid_value(v):
self.log.debug('Discard invalid tag value: %s', v)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
tags.append((k, v))
data['tags'] = tags
for k in list(iter(data)):
if k in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(k)
if not value:
self.log.debug('Ignored empty interface value: %s', k)
continue
try:
interface = get_interface(k)
except ValueError:
self.log.debug('Ignored unknown attribute: %s', k)
data['errors'].append({
'type': EventError.INVALID_ATTRIBUTE,
'name': k,
})
continue
if type(value) != dict:
# HACK(dcramer): the exception/breadcrumbs interface supports a
# list as the value. We should change this in a new protocol
# version.
if type(value) in (list, tuple):
value = {'values': value}
else:
self.log.debug(
'Invalid parameter for value: %s (%r)', k, type(value))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, InterfaceValidationError):
log = self.log.debug
else:
log = self.log.error
log('Discarded invalid value for interface: %s (%r)', k, value,
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
# TODO(dcramer): ideally this logic would happen in normalize, but today
# we don't do "validation" there (create errors)
# message is coerced to an interface, as its used for pure
# index of searchable strings
# See GH-3248
message = data.pop('message', None)
if message:
if 'sentry.interfaces.Message' not in data:
value = {
'message': message,
}
elif not data['sentry.interfaces.Message'].get('formatted'):
value = data['sentry.interfaces.Message']
value['formatted'] = message
else:
value = None
if value is not None:
k = 'sentry.interfaces.Message'
interface = get_interface(k)
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, InterfaceValidationError):
log = self.log.debug
else:
log = self.log.error
log('Discarded invalid value for interface: %s (%r)', k, value,
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
level = data.get('level') or DEFAULT_LOG_LEVEL
if isinstance(level, six.string_types) and not level.isdigit():
# assume it's something like 'warning'
try:
data['level'] = LOG_LEVELS_MAP[level]
except KeyError as e:
self.log.debug(
'Discarded invalid logger value: %s', level)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'level',
'value': level,
})
data['level'] = LOG_LEVELS_MAP.get(
DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL)
if data.get('release'):
data['release'] = six.text_type(data['release'])
if len(data['release']) > 64:
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'release',
'value': data['release'],
})
del data['release']
if data.get('dist'):
data['dist'] = six.text_type(data['dist']).strip()
if not data.get('release'):
data['dist'] = None
elif len(data['dist']) > 64:
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'dist',
'value': data['dist'],
})
del data['dist']
elif _dist_re.match(data['dist']) is None:
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'dist',
'value': data['dist'],
})
del data['dist']
if data.get('environment'):
data['environment'] = six.text_type(data['environment'])
if len(data['environment']) > 64:
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'environment',
'value': data['environment'],
})
del data['environment']
if data.get('time_spent'):
try:
data['time_spent'] = int(data['time_spent'])
except (ValueError, TypeError):
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'time_spent',
'value': data['time_spent'],
})
del data['time_spent']
else:
if data['time_spent'] > BoundedIntegerField.MAX_VALUE:
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'time_spent',
'value': data['time_spent'],
})
del data['time_spent']
return data
def validate_data(self, project, data):
# TODO(dcramer): move project out of the data packet
data['project'] = project.id
data['errors'] = []
if not data.get('message'):
data['message'] = '<no message value>'
elif not isinstance(data['message'], six.string_types):
raise APIForbidden('Invalid value for message')
if data.get('culprit'):
if not isinstance(data['culprit'], six.string_types):
raise APIForbidden('Invalid value for culprit')
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif not isinstance(data['event_id'], six.string_types):
raise APIForbidden('Invalid value for event_id')
if len(data['event_id']) > 32:
self.log.info(
'Discarded value for event_id due to length (%d chars)',
len(data['event_id']))
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'event_id',
'value': data['event_id'],
})
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
self._process_data_timestamp(data)
except InvalidTimestamp as e:
self.log.info(
'Discarded invalid value for timestamp: %r',
data['timestamp'], exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'timestamp',
'value': data['timestamp'],
})
del data['timestamp']
if 'fingerprint' in data:
try:
self._process_fingerprint(data)
except InvalidFingerprint as e:
self.log.info(
'Discarded invalid value for fingerprint: %r',
data['fingerprint'], exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'fingerprint',
'value': data['fingerprint'],
})
del data['fingerprint']
if 'platform' not in data or data['platform'] not in VALID_PLATFORMS:
data['platform'] = 'other'
if data.get('modules') and type(data['modules']) != dict:
self.log.info(
'Discarded invalid type for modules: %s',
type(data['modules']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'modules',
'value': data['modules'],
})
del data['modules']
if data.get('extra') is not None and type(data['extra']) != dict:
self.log.info(
'Discarded invalid type for extra: %s',
type(data['extra']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'extra',
'value': data['extra'],
})
del data['extra']
if data.get('tags') is not None:
if type(data['tags']) == dict:
data['tags'] = data['tags'].items()
elif not isinstance(data['tags'], (list, tuple)):
self.log.info(
'Discarded invalid type for tags: %s', type(data['tags']))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': data['tags'],
})
del data['tags']
if data.get('tags'):
# remove any values which are over 32 characters
tags = []
for pair in data['tags']:
try:
k, v = pair
except ValueError:
self.log.info('Discarded invalid tag value: %r', pair)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not isinstance(k, six.string_types):
try:
k = six.text_type(k)
except Exception:
self.log.info('Discarded invalid tag key: %r', type(k))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not isinstance(v, six.string_types):
try:
v = six.text_type(v)
except Exception:
self.log.info('Discarded invalid tag value: %s=%r',
k, type(v))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH:
self.log.info('Discarded invalid tag: %s=%s', k, v)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
# support tags with spaces by converting them
k = k.replace(' ', '-')
if TagKey.is_reserved_key(k):
self.log.info('Discarding reserved tag key: %s', k)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
if not TagKey.is_valid_key(k):
self.log.info('Discarded invalid tag key: %s', k)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'tags',
'value': pair,
})
continue
tags.append((k, v))
data['tags'] = tags
for k in data.keys():
if k in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(k)
if not value:
self.log.info('Ignored empty interface value: %s', k)
continue
try:
interface = get_interface(k)
except ValueError:
self.log.info('Ignored unknown attribute: %s', k)
data['errors'].append({
'type': EventError.INVALID_ATTRIBUTE,
'name': k,
})
continue
if type(value) != dict:
# HACK(dcramer): the exception/breadcrumbs interface supports a
# list as the value. We should change this in a new protocol
# version.
if type(value) in (list, tuple):
value = {'values': value}
else:
self.log.info(
'Invalid parameter for value: %s (%r)', k, type(value))
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, InterfaceValidationError):
log = self.log.info
else:
log = self.log.error
log('Discarded invalid value for interface: %s (%r)', k, value,
exc_info=True)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': k,
'value': value,
})
level = data.get('level') or DEFAULT_LOG_LEVEL
if isinstance(level, six.string_types) and not level.isdigit():
# assume it's something like 'warning'
try:
data['level'] = LOG_LEVEL_REVERSE_MAP[level]
except KeyError as e:
self.log.info(
'Discarded invalid logger value: %s', level)
data['errors'].append({
'type': EventError.INVALID_DATA,
'name': 'level',
'value': level,
})
data['level'] = LOG_LEVEL_REVERSE_MAP.get(
DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL)
if data.get('release'):
data['release'] = unicode(data['release'])
if len(data['release']) > 64:
data['errors'].append({
'type': EventError.VALUE_TOO_LONG,
'name': 'release',
'value': data['release'],
})
del data['release']
return data
def validate_data(self, project, data):
# TODO(dcramer): move project out of the data packet
data["project"] = project.id
data["errors"] = []
if not data.get("message"):
data["message"] = "<no message value>"
elif not isinstance(data["message"], six.string_types):
raise APIForbidden("Invalid value for message")
if data.get("culprit"):
if not isinstance(data["culprit"], six.string_types):
raise APIForbidden("Invalid value for culprit")
if not data.get("event_id"):
data["event_id"] = uuid.uuid4().hex
elif not isinstance(data["event_id"], six.string_types):
raise APIForbidden("Invalid value for event_id")
if len(data["event_id"]) > 32:
self.log.info("Discarded value for event_id due to length (%d chars)", len(data["event_id"]))
data["errors"].append({"type": EventError.VALUE_TOO_LONG, "name": "event_id", "value": data["event_id"]})
data["event_id"] = uuid.uuid4().hex
if "timestamp" in data:
try:
self._process_data_timestamp(data)
except InvalidTimestamp as e:
self.log.info("Discarded invalid value for timestamp: %r", data["timestamp"], exc_info=True)
data["errors"].append(
{"type": EventError.INVALID_DATA, "name": "timestamp", "value": data["timestamp"]}
)
del data["timestamp"]
if "fingerprint" in data:
try:
self._process_fingerprint(data)
except InvalidFingerprint as e:
self.log.info("Discarded invalid value for fingerprint: %r", data["fingerprint"], exc_info=True)
data["errors"].append(
{"type": EventError.INVALID_DATA, "name": "fingerprint", "value": data["fingerprint"]}
)
if "platform" not in data or data["platform"] not in VALID_PLATFORMS:
data["platform"] = "other"
if data.get("modules") and type(data["modules"]) != dict:
self.log.info("Discarded invalid type for modules: %s", type(data["modules"]))
data["errors"].append({"type": EventError.INVALID_DATA, "name": "modules", "value": data["modules"]})
del data["modules"]
if data.get("extra") is not None and type(data["extra"]) != dict:
self.log.info("Discarded invalid type for extra: %s", type(data["extra"]))
data["errors"].append({"type": EventError.INVALID_DATA, "name": "extra", "value": data["extra"]})
del data["extra"]
if data.get("tags") is not None:
if type(data["tags"]) == dict:
data["tags"] = data["tags"].items()
elif not isinstance(data["tags"], (list, tuple)):
self.log.info("Discarded invalid type for tags: %s", type(data["tags"]))
data["errors"].append({"type": EventError.INVALID_DATA, "name": "tags", "value": data["tags"]})
del data["tags"]
if data.get("tags"):
# remove any values which are over 32 characters
tags = []
for pair in data["tags"]:
try:
k, v = pair
except ValueError:
self.log.info("Discarded invalid tag value: %r", pair)
data["errors"].append({"type": EventError.INVALID_DATA, "name": "tags", "value": pair})
continue
if not isinstance(k, six.string_types):
try:
k = six.text_type(k)
except Exception:
self.log.info("Discarded invalid tag key: %r", type(k))
data["errors"].append({"type": EventError.INVALID_DATA, "name": "tags", "value": pair})
continue
if not isinstance(v, six.string_types):
try:
v = six.text_type(v)
except Exception:
self.log.info("Discarded invalid tag value: %s=%r", k, type(v))
data["errors"].append({"type": EventError.INVALID_DATA, "name": "tags", "value": pair})
continue
if len(k) > MAX_TAG_KEY_LENGTH or len(v) > MAX_TAG_VALUE_LENGTH:
self.log.info("Discarded invalid tag: %s=%s", k, v)
data["errors"].append({"type": EventError.INVALID_DATA, "name": "tags", "value": pair})
continue
# support tags with spaces by converting them
k = k.replace(" ", "-")
if not TagKey.is_valid_key(k):
self.log.info("Discarded invalid tag key: %s", k)
data["errors"].append({"type": EventError.INVALID_DATA, "name": "tags", "value": pair})
continue
tags.append((k, v))
data["tags"] = tags
for k in data.keys():
if k in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(k)
if not value:
self.log.info("Ignored empty interface value: %s", k)
continue
try:
interface = get_interface(k)
except ValueError:
self.log.info("Ignored unknown attribute: %s", k)
data["errors"].append({"type": EventError.INVALID_ATTRIBUTE, "name": k})
continue
if type(value) != dict:
# HACK(dcramer): the exception interface supports a list as the
# value. We should change this in a new protocol version.
if type(value) in (list, tuple):
value = {"values": value}
else:
self.log.info("Invalid parameter for value: %s (%r)", k, type(value))
data["errors"].append({"type": EventError.INVALID_DATA, "name": k, "value": value})
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception as e:
if isinstance(e, InterfaceValidationError):
log = self.log.info
else:
log = self.log.error
log("Discarded invalid value for interface: %s (%r)", k, value, exc_info=True)
data["errors"].append({"type": EventError.INVALID_DATA, "name": k, "value": value})
level = data.get("level") or DEFAULT_LOG_LEVEL
if isinstance(level, six.string_types) and not level.isdigit():
# assume it's something like 'warning'
try:
data["level"] = LOG_LEVEL_REVERSE_MAP[level]
except KeyError as e:
self.log.info("Discarded invalid logger value: %s", level)
data["errors"].append({"type": EventError.INVALID_DATA, "name": "level", "value": level})
data["level"] = LOG_LEVEL_REVERSE_MAP.get(DEFAULT_LOG_LEVEL, DEFAULT_LOG_LEVEL)
if data.get("release"):
data["release"] = unicode(data["release"])
if len(data["release"]) > 64:
data["errors"].append({"type": EventError.VALUE_TOO_LONG, "name": "release", "value": data["release"]})
del data["release"]
return data
def normalize(self):
# TODO(dcramer): store http.env.REMOTE_ADDR as user.ip
# First we pull out our top-level (non-data attr) kwargs
data = self.data
if not isinstance(data.get("level"), (six.string_types, int)):
data["level"] = logging.ERROR
elif data["level"] not in LOG_LEVELS:
data["level"] = logging.ERROR
if not data.get("logger"):
data["logger"] = DEFAULT_LOGGER_NAME
else:
logger = trim(data["logger"].strip(), 64)
if TagKey.is_valid_key(logger):
data["logger"] = logger
else:
data["logger"] = DEFAULT_LOGGER_NAME
if data.get("platform"):
data["platform"] = trim(data["platform"], 64)
current_timestamp = timezone.now()
timestamp = data.get("timestamp")
if not timestamp:
timestamp = current_timestamp
if isinstance(timestamp, datetime):
# We must convert date to local time so Django doesn't mess it up
# based on TIME_ZONE
if settings.TIME_ZONE:
if not timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=timezone.utc)
elif timezone.is_aware(timestamp):
timestamp = timestamp.replace(tzinfo=None)
timestamp = float(timestamp.strftime("%s"))
data["timestamp"] = timestamp
data["received"] = float(timezone.now().strftime("%s"))
if not data.get("event_id"):
data["event_id"] = uuid4().hex
data.setdefault("message", "")
data.setdefault("culprit", None)
data.setdefault("server_name", None)
data.setdefault("site", None)
data.setdefault("checksum", None)
data.setdefault("fingerprint", None)
data.setdefault("platform", None)
data.setdefault("environment", None)
data.setdefault("extra", {})
data.setdefault("errors", [])
tags = data.get("tags")
if not tags:
tags = []
# full support for dict syntax
elif isinstance(tags, dict):
tags = tags.items()
# prevent [tag, tag, tag] (invalid) syntax
elif not all(len(t) == 2 for t in tags):
tags = []
else:
tags = list(tags)
data["tags"] = []
for key, value in tags:
key = six.text_type(key).strip()
value = six.text_type(value).strip()
if not (key and value):
continue
data["tags"].append((key, value))
if not isinstance(data["extra"], dict):
# throw it away
data["extra"] = {}
trim_dict(data["extra"], max_size=settings.SENTRY_MAX_EXTRA_VARIABLE_SIZE)
# TODO(dcramer): more of validate data needs stuffed into the manager
for key in data.keys():
if key in CLIENT_RESERVED_ATTRS:
continue
value = data.pop(key)
try:
interface = get_interface(key)()
except ValueError:
continue
try:
inst = interface.to_python(value)
data[inst.get_path()] = inst.to_json()
except Exception:
pass
# the SDKs currently do not describe event types, and we must infer
# them from available attributes
data["type"] = eventtypes.infer(data).key
data["version"] = self.version
# TODO(dcramer): find a better place for this logic
exception = data.get("sentry.interfaces.Exception")
stacktrace = data.get("sentry.interfaces.Stacktrace")
if exception and len(exception["values"]) == 1 and stacktrace:
exception["values"][0]["stacktrace"] = stacktrace
del data["sentry.interfaces.Stacktrace"]
if "sentry.interfaces.Http" in data:
try:
ip_address = validate_ip(
data["sentry.interfaces.Http"].get("env", {}).get("REMOTE_ADDR"), required=False
)
except ValueError:
ip_address = None
if ip_address:
data.setdefault("sentry.interfaces.User", {})
data["sentry.interfaces.User"].setdefault("ip_address", ip_address)
if data["culprit"]:
data["culprit"] = trim(data["culprit"], MAX_CULPRIT_LENGTH)
if data["message"]:
data["message"] = trim(data["message"], settings.SENTRY_MAX_MESSAGE_LENGTH)
return data
