def start(ctx, service, bind, workers, debug, upgrade, noinput):
"Start running a service."
if bind:
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
else:
host, port = None, None
if upgrade:
click.echo('Performing upgrade before service startup...')
from sentry.runner import call_command
call_command(
'sentry.runner.commands.upgrade.upgrade',
verbosity=0, noinput=noinput,
)
click.echo('Running service: %r' % service)
# remove command line arguments to avoid optparse failures with service code
# that calls call_command which reparses the command line, and if --noupgrade is supplied
# a parse error is thrown
import sys
sys.argv = sys.argv[:1]
from sentry.utils.imports import import_string
import_string(SERVICES[service])(
debug=debug,
host=host,
port=port,
workers=workers,
).run()
def start(ctx, service, bind, workers, debug, upgrade, noinput):
"Start running a service."
if bind:
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
else:
host, port = None, None
if upgrade:
click.echo('Performing upgrade before service startup...')
from sentry.runner import call_command
call_command(
'sentry.runner.commands.upgrade.upgrade',
verbosity=0, noinput=noinput,
)
click.echo('Running service: %r' % service)
# remove command line arguments to avoid optparse failures with service code
# that calls call_command which reparses the command line, and if --noupgrade is supplied
# a parse error is thrown
import sys
sys.argv = sys.argv[:1]
from sentry.utils.imports import import_string
import_string(SERVICES[service])(
debug=debug,
host=host,
port=port,
workers=workers,
).run()
def mocked_attachment_cache(request, mock_client):
class RbCluster(object):
def get_routing_client(self):
return mock_client
if request.param == "rb":
with mock.patch("sentry.cache.redis.get_cluster_from_options",
return_value=(RbCluster(), {})) as cluster_get:
attachment_cache = import_string(
"sentry.attachments.redis.RbAttachmentCache")(hosts=[])
cluster_get.assert_any_call("SENTRY_CACHE_OPTIONS", {"hosts": []})
assert isinstance(attachment_cache.inner, RbCache)
elif request.param == "rediscluster":
with mock.patch("sentry.utils.redis.redis_clusters.get",
return_value=mock_client) as cluster_get:
attachment_cache = import_string(
"sentry.attachments.redis.RedisClusterAttachmentCache")()
cluster_get.assert_any_call("rc-short")
assert isinstance(attachment_cache.inner, RedisClusterCache)
else:
assert False
assert attachment_cache.inner.client is mock_client
yield attachment_cache
def validate_dependency(settings, dependency_type, dependency, package):
try:
import_string(package)
except ImportError:
msg = ConfigurationError.get_error_message(
"%s %s" % (dependency_type, dependency), package)
reraise_as(ConfigurationError(msg))
def validate_dependency(settings, dependency_type, dependency, package):
try:
import_string(package)
except ImportError:
msg = ConfigurationError.get_error_message(
"%s %s" % (dependency_type, dependency), package)
reraise_as(ConfigurationError(msg))
def register_plugins(settings):
from pkg_resources import iter_entry_points
from sentry.plugins import plugins
# entry_points={
# 'sentry.plugins': [
# 'phabricator = sentry_phabricator.plugins:PhabricatorPlugin'
# ],
# },
for ep in iter_entry_points('sentry.plugins'):
try:
plugin = ep.load()
except Exception:
import traceback
click.echo("Failed to load plugin %r:\n%s" %
(ep.name, traceback.format_exc()),
err=True)
else:
plugins.register(plugin)
for plugin in plugins.all(version=None):
init_plugin(plugin)
from sentry import integrations
from sentry.utils.imports import import_string
for integration_path in settings.SENTRY_DEFAULT_INTEGRATIONS:
try:
integration_cls = import_string(integration_path)
except Exception:
import traceback
click.echo("Failed to load integration %r:\n%s" %
(integration_path, traceback.format_exc()),
err=True)
else:
integrations.register(integration_cls)
def __init__(self, target, callback):
target, attr = target.rsplit('.', 1)
target = import_string(target)
self.func = getattr(target, attr)
self.target = target
self.attr = attr
self.callback = callback
def process(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, '1', nx=True, ex=10):
metrics.incr('buffer.revoked', tags={'reason': 'locked'})
self.logger.info('Skipped process on %s; unable to get lock', key)
return
conn = self.cluster.get_local_client_for_key(key)
pipe = conn.pipeline()
pipe.hgetall(key)
pipe.zrem(self.pending_key, key)
pipe.delete(key)
values = pipe.execute()[0]
if not values:
metrics.incr('buffer.revoked', tags={'reason': 'empty'})
self.logger.info('Skipped process on %s; no values found', key)
return
model = import_string(values['m'])
filters = pickle.loads(values['f'])
incr_values = {}
extra_values = {}
for k, v in values.iteritems():
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters, extra_values)
def process(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, '1', nx=True, ex=10):
metrics.incr('buffer.revoked', tags={'reason': 'locked'})
self.logger.info('Skipped process on %s; unable to get lock', key)
return
conn = self.cluster.get_local_client_for_key(key)
pipe = conn.pipeline()
pipe.hgetall(key)
pipe.zrem(self.pending_key, key)
pipe.delete(key)
values = pipe.execute()[0]
if not values:
metrics.incr('buffer.revoked', tags={'reason': 'empty'})
self.logger.info('Skipped process on %s; no values found', key)
return
model = import_string(values['m'])
filters = pickle.loads(values['f'])
incr_values = {}
extra_values = {}
for k, v in values.iteritems():
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters,
extra_values)
def process(self, key):
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not self.conn.setnx(lock_key, '1'):
return
self.conn.expire(lock_key, 10)
with self.conn.map() as conn:
values = conn.hgetall(key)
conn.delete(key)
if not values:
return
model = import_string(values['m'])
filters = pickle.loads(values['f'])
incr_values = {}
extra_values = {}
for k, v in values.iteritems():
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters,
extra_values)
def process(self, key):
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not self.conn.setnx(lock_key, '1'):
return
self.conn.expire(lock_key, 10)
with self.conn.map() as conn:
values = conn.hgetall(key)
conn.delete(key)
if not values:
return
model = import_string(values['m'])
filters = pickle.loads(values['f'])
incr_values = {}
extra_values = {}
for k, v in values.iteritems():
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters, extra_values)
def process(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, '1', nx=True, ex=10):
return
with self.cluster.map() as conn:
values = conn.hgetall(key)
conn.delete(key)
if not values.value:
return
model = import_string(values.value['m'])
filters = pickle.loads(values.value['f'])
incr_values = {}
extra_values = {}
for k, v in values.value.iteritems():
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters, extra_values)
def validate_data(project, data, client=None):
ensure_valid_project_id(project, data)
if not data.get('message'):
data['message'] = '<no message value>'
elif len(data['message']) > MAX_MESSAGE_LENGTH:
raise InvalidData('Value \'message\' is too long. Input is %d chars, max is %s.' % (
len(data['message']), MAX_MESSAGE_LENGTH))
if data.get('culprit') and len(data['culprit']) > 200:
raise InvalidData('Value \'culprit\' is too long. Input is %d chars, max is %s.' % (
len(data['culprit']), MAX_CULPRIT_LENGTH))
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif len(data['event_id']) > 32:
raise InvalidData('Invalid value for \'event_id\': must be a 32 character identifier')
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp:
# Log the error, remove the timestamp, and continue
logger.info('Client %r passed an invalid value for timestamp %r',
client or '<unknown client>', data['timestamp'], extra={'request': env.request})
del data['timestamp']
if data.get('modules') and type(data['modules']) != dict:
raise InvalidData('Invalid type for \'modules\': must be a mapping')
for k in data.keys():
if k in RESERVED_FIELDS:
continue
if not data[k]:
logger.info('Ignoring empty interface %r passed by client %r',
k, client or '<unknown client>', extra={'request': env.request})
del data[k]
continue
import_path = INTERFACE_ALIASES.get(k, k)
if '.' not in import_path:
logger.info('Ignoring unknown attribute %r passed by client %r',
k, client or '<unknown client>', extra={'request': env.request})
del data[k]
continue
try:
interface = import_string(import_path)
except (ImportError, AttributeError), e:
raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))
try:
inst = interface(**data.pop(k))
inst.validate()
data[import_path] = inst.serialize()
except AssertionError, e:
raise InvalidData('Unable to validate interface, %r: %s' % (k, e))
def _process_single_incr(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, "1", nx=True, ex=10):
metrics.incr("buffer.revoked", tags={"reason": "locked"}, skip_internal=False)
self.logger.debug("buffer.revoked.locked", extra={"redis_key": key})
return
pending_key = self._make_pending_key_from_key(key)
try:
conn = self.cluster.get_local_client_for_key(key)
pipe = conn.pipeline()
pipe.hgetall(key)
pipe.zrem(pending_key, key)
pipe.delete(key)
values = pipe.execute()[0]
# XXX(python3): In python2 this isn't as important since redis will
# return string tyes (be it, byte strings), but in py3 we get bytes
# back, and really we just want to deal with keys as strings.
values = {force_text(k): v for k, v in six.iteritems(values)}
if not values:
metrics.incr("buffer.revoked", tags={"reason": "empty"}, skip_internal=False)
self.logger.debug("buffer.revoked.empty", extra={"redis_key": key})
return
# XXX(py3): Note that ``import_string`` explicitly wants a str in
# python2, so we'll decode (for python3) and then translate back to
# a byte string (in python2) for import_string.
model = import_string(str(values.pop("m").decode("utf-8"))) # NOQA
if values["f"].startswith(b"{"):
filters = self._load_values(json.loads(values.pop("f").decode("utf-8")))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
filters = pickle.loads(values.pop("f"))
incr_values = {}
extra_values = {}
signal_only = None
for k, v in six.iteritems(values):
if k.startswith("i+"):
incr_values[k[2:]] = int(v)
elif k.startswith("e+"):
if v.startswith(b"["):
extra_values[k[2:]] = self._load_value(json.loads(v.decode("utf-8")))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
extra_values[k[2:]] = pickle.loads(v)
elif k == "s":
signal_only = bool(int(v)) # Should be 1 if set
super(RedisBuffer, self).process(model, incr_values, filters, extra_values, signal_only)
finally:
client.delete(lock_key)
def multiprocess_worker(task_queue):
# Configure within each Process
import logging
from sentry.utils.imports import import_string
logger = logging.getLogger('sentry.cleanup')
configured = False
while True:
j = task_queue.get()
if j == _STOP_WORKER:
task_queue.task_done()
return
# On first task, configure Sentry environment
if not configured:
from sentry.runner import configure
configure()
from sentry import models
from sentry import deletions
from sentry import similarity
skip_models = [
# Handled by other parts of cleanup
models.Event,
models.EventMapping,
models.EventAttachment,
models.UserReport,
models.Group,
models.GroupEmailThread,
models.GroupRuleStatus,
models.GroupHashTombstone,
# Handled by TTL
similarity.features,
] + [b[0] for b in EXTRA_BULK_QUERY_DELETES]
configured = True
model, chunk = j
model = import_string(model)
try:
task = deletions.get(
model=model,
query={'id__in': chunk},
skip_models=skip_models,
transaction_id=uuid4().hex,
)
while True:
if not task.chunk():
break
except Exception as e:
logger.exception(e)
finally:
task_queue.task_done()
def start(ctx, service, bind, workers, upgrade, noinput):
"DEPRECATED see `sentry run` instead."
from sentry.runner.initializer import show_big_error
show_big_error(
[
'`sentry start%s` is deprecated.' % (' ' + service if 'http' in sys.argv else ''),
'Use `sentry run %s` instead.' % {
'http': 'web'
}.get(service, service),
]
)
if bind:
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
else:
host, port = None, None
if upgrade:
click.echo('Performing upgrade before service startup...')
from sentry.runner import call_command
call_command(
'sentry.runner.commands.upgrade.upgrade',
verbosity=0,
noinput=noinput,
)
click.echo('Running service: %r' % service)
# remove command line arguments to avoid optparse failures with service code
# that calls call_command which reparses the command line, and if --noupgrade is supplied
# a parse error is thrown
sys.argv = sys.argv[:1]
from sentry.utils.imports import import_string
import_string(SERVICES[service])(
host=host,
port=port,
workers=workers,
).run()
def start(ctx, service, bind, workers, upgrade, noinput):
"DEPRECATED see `sentry run` instead."
from sentry.runner.initializer import show_big_error
show_big_error(
[
'`sentry start%s` is deprecated.' % (' ' + service if 'http' in sys.argv else ''),
'Use `sentry run %s` instead.' % {
'http': 'web'
}.get(service, service),
]
)
if bind:
if ':' in bind:
host, port = bind.split(':', 1)
port = int(port)
else:
host = bind
port = None
else:
host, port = None, None
if upgrade:
click.echo('Performing upgrade before service startup...')
from sentry.runner import call_command
call_command(
'sentry.runner.commands.upgrade.upgrade',
verbosity=0,
noinput=noinput,
)
click.echo('Running service: %r' % service)
# remove command line arguments to avoid optparse failures with service code
# that calls call_command which reparses the command line, and if --noupgrade is supplied
# a parse error is thrown
sys.argv = sys.argv[:1]
from sentry.utils.imports import import_string
import_string(SERVICES[service])(
host=host,
port=port,
workers=workers,
).run()
def iter_interfaces():
rv = {}
for name, import_path in settings.SENTRY_INTERFACES.iteritems():
rv.setdefault(import_path, []).append(name)
for import_path, keys in rv.iteritems():
iface = import_string(import_path)
yield iface, keys
def init_registry():
from sentry.constants import SENTRY_RULES
from sentry.utils.imports import import_string
registry = RuleRegistry()
for rule in SENTRY_RULES:
cls = import_string(rule)
registry.add(cls)
return registry
def iter_interfaces():
rv = {}
for name, import_path in six.iteritems(settings.SENTRY_INTERFACES):
rv.setdefault(import_path, []).append(name)
for import_path, keys in six.iteritems(rv):
iface = import_string(import_path)
yield iface, keys
def _process_single_incr(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, "1", nx=True, ex=10):
metrics.incr("buffer.revoked",
tags={"reason": "locked"},
skip_internal=False)
self.logger.debug("buffer.revoked.locked",
extra={"redis_key": key})
return
pending_key = self._make_pending_key_from_key(key)
try:
conn = self.cluster.get_local_client_for_key(key)
pipe = conn.pipeline()
pipe.hgetall(key)
pipe.zrem(pending_key, key)
pipe.delete(key)
values = pipe.execute()[0]
if not values:
metrics.incr("buffer.revoked",
tags={"reason": "empty"},
skip_internal=False)
self.logger.debug("buffer.revoked.empty",
extra={"redis_key": key})
return
model = import_string(values.pop("m"))
if values["f"].startswith("{"):
filters = self._load_values(json.loads(values.pop("f")))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
filters = pickle.loads(values.pop("f"))
incr_values = {}
extra_values = {}
signal_only = None
for k, v in six.iteritems(values):
if k.startswith("i+"):
incr_values[k[2:]] = int(v)
elif k.startswith("e+"):
if v.startswith("["):
extra_values[k[2:]] = self._load_value(json.loads(v))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
extra_values[k[2:]] = pickle.loads(v)
elif k == "s":
signal_only = bool(int(v)) # Should be 1 if set
super(RedisBuffer, self).process(model, incr_values, filters,
extra_values, signal_only)
finally:
client.delete(lock_key)
def __init__(self, target, callback):
target, attr = target.rsplit(".", 1)
target = import_string(target)
self.target = target
self.attr = attr
self.callback = callback
self._lock = Lock()
with self._lock:
self.func = getattr(target, attr)
def get_interface(name):
if name not in settings.SENTRY_ALLOWED_INTERFACES:
raise ValueError
try:
interface = import_string(name)
except Exception:
raise ValueError('Unable to load interface: %s' % (name,))
return interface
def get_interface(name):
if name not in settings.SENTRY_ALLOWED_INTERFACES:
raise ValueError
try:
interface = import_string(name)
except Exception:
raise ValueError('Unable to load interface: %s' % (name, ))
return interface
def __init__(self, backends, read_selector=random.choice, **kwargs):
assert backends, "you should provide at least one backend"
self.backends = []
for backend, backend_options in backends:
if isinstance(backend, six.string_types):
backend = import_string(backend)
self.backends.append(backend(**backend_options))
self.read_selector = read_selector
super(MultiNodeStorage, self).__init__(**kwargs)
def get_password_validators(validator_config):
validators = []
for validator in validator_config:
try:
klass = import_string(validator['NAME'])
except ImportError:
msg = "The module in NAME could not be imported: %s. Check your AUTH_PASSWORD_VALIDATORS setting."
raise ImproperlyConfigured(msg % validator['NAME'])
validators.append(klass(**validator.get('OPTIONS', {})))
return validators
def _process_single_incr(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, '1', nx=True, ex=10):
metrics.incr('buffer.revoked',
tags={'reason': 'locked'},
skip_internal=False)
self.logger.debug('buffer.revoked.locked',
extra={'redis_key': key})
return
pending_key = self._make_pending_key_from_key(key)
try:
conn = self.cluster.get_local_client_for_key(key)
pipe = conn.pipeline()
pipe.hgetall(key)
pipe.zrem(pending_key, key)
pipe.delete(key)
values = pipe.execute()[0]
if not values:
metrics.incr('buffer.revoked',
tags={'reason': 'empty'},
skip_internal=False)
self.logger.debug('buffer.revoked.empty',
extra={'redis_key': key})
return
model = import_string(values.pop('m'))
if values['f'].startswith('{'):
filters = self._load_values(json.loads(values.pop('f')))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
filters = pickle.loads(values.pop('f'))
incr_values = {}
extra_values = {}
for k, v in six.iteritems(values):
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
if v.startswith('['):
extra_values[k[2:]] = self._load_value(json.loads(v))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters,
extra_values)
finally:
client.delete(lock_key)
def get_instance(attribute, options, dangerous=()):
value = getattr(settings, attribute)
cls = import_string(value)
if cls in dangerous:
warnings.warn(
warnings.UnsupportedBackend(
u'The {!r} backend for {} is not recommended '
'for production use.'.format(value, attribute)))
return cls(**options)
def get_password_validators(validator_config):
validators = []
for validator in validator_config:
try:
klass = import_string(validator["NAME"])
except ImportError:
msg = "The module in NAME could not be imported: %s. Check your AUTH_PASSWORD_VALIDATORS setting."
raise ImproperlyConfigured(msg % validator["NAME"])
validators.append(klass(**validator.get("OPTIONS", {})))
return validators
def call_command(name, obj=None, **kwargs):
try:
command = import_string(name)
except (ImportError, AttributeError):
raise UnknownCommand(name)
with command.make_context('sentry', [], obj=obj or {}) as ctx:
ctx.params.update(kwargs)
try:
command.invoke(ctx)
except click.Abort:
click.echo('Aborted!', err=True)
def get_interface(name):
try:
import_path = settings.SENTRY_INTERFACES[name]
except KeyError:
raise ValueError('Invalid interface name: %s' % (name, ))
try:
interface = import_string(import_path)
except Exception:
raise ValueError('Unable to load interface: %s' % (name, ))
return interface
def call_command(name, obj=None, **kwargs):
try:
command = import_string(name)
except (ImportError, AttributeError):
raise UnknownCommand(name)
with command.make_context("sentry", [], obj=obj or {}) as ctx:
ctx.params.update(kwargs)
try:
command.invoke(ctx)
except click.Abort:
click.echo("Aborted!", err=True)
def get_interface(name):
try:
import_path = settings.SENTRY_INTERFACES[name]
except KeyError:
raise ValueError('Invalid interface name: %s' % (name, ))
try:
interface = import_string(import_path)
except Exception:
raise ValueError('Unable to load interface: %s' % (name, ))
return interface
def register_plugins(settings, test_plugins=False):
from pkg_resources import iter_entry_points
from sentry.plugins.base import plugins
# entry_points={
# 'sentry.plugins': [
# 'phabricator = sentry_phabricator.plugins:PhabricatorPlugin'
# ],
# },
entry_points = [
"sentry.new_plugins",
"sentry.test_only_plugins" if test_plugins else "sentry.plugins",
]
for entry_point in entry_points:
for ep in iter_entry_points(entry_point):
try:
plugin = ep.load()
except Exception:
import traceback
click.echo("Failed to load plugin %r:\n%s" %
(ep.name, traceback.format_exc()),
err=True)
else:
plugins.register(plugin)
for plugin in plugins.all(version=None):
init_plugin(plugin)
from sentry import integrations
from sentry.utils.imports import import_string
for integration_path in settings.SENTRY_DEFAULT_INTEGRATIONS:
try:
integration_cls = import_string(integration_path)
except Exception:
import traceback
click.echo(
"Failed to load integration %r:\n%s" %
(integration_path, traceback.format_exc()),
err=True,
)
else:
integrations.register(integration_cls)
for integration in integrations.all():
try:
integration.setup()
except AttributeError:
pass
def interfaces(self):
result = []
for key, data in self.data.iteritems():
if '.' not in key:
continue
try:
cls = import_string(key)
except ImportError:
pass # suppress invalid interfaces
value = cls(**data)
result.append((value.score, key, value))
return SortedDict((k, v) for _, k, v in sorted(result, key=lambda x: x[0], reverse=True))
def get_interface(name):
try:
name = get_canonical_name(name)
import_path = settings.SENTRY_INTERFACES[name]
except KeyError:
raise ValueError(f"Invalid interface name: {name}")
try:
interface = import_string(import_path)
except Exception:
raise ValueError(f"Unable to load interface: {name}")
return interface
def get_instance(attribute, options, dangerous=()):
value = getattr(settings, attribute)
cls = import_string(value)
if cls in dangerous:
warnings.warn(
warnings.UnsupportedBackend(
u'The {!r} backend for {} is not recommended '
'for production use.'.format(value, attribute)
)
)
return cls(**options)
def interfaces(self):
result = []
for key, data in self.data.iteritems():
if '.' not in key:
continue
try:
cls = import_string(key)
except ImportError:
pass # suppress invalid interfaces
value = cls(**data)
result.append((value.score, key, value))
return SortedDict((k, v) for _, k, v in sorted(result, key=lambda x: x[0], reverse=True))
def validate_data(project, data, client=None):
ensure_valid_project_id(project, data)
if not data.get('message'):
data['message'] = '<no message value>'
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif len(data['event_id']) > 32:
raise InvalidData(
'Invalid value for \'event_id\': must be a 32 character identifier'
)
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp:
# Log the error, remove the timestamp, and continue
logger.error(
'Client %r passed an invalid value for timestamp %r',
client or '<unknown client>',
data['timestamp'],
)
del data['timestamp']
if data.get('modules') and type(data['modules']) != dict:
raise InvalidData('Invalid type for \'modules\': must be a mapping')
for k, v in data.items():
if k in RESERVED_FIELDS:
continue
if '.' not in k:
logger.error(
'Ignoring unknown attribute %r passed by client %r',
k,
client or '<unknown client>',
)
del data[k]
continue
try:
interface = import_string(k)
except (ImportError, AttributeError), e:
raise InvalidInterface('%r is not a valid interface name: %s' %
(k, e))
try:
data[k] = interface(**v).serialize()
except Exception, e:
raise InvalidData('Unable to validate interface, %r: %s' % (k, e))
def __init__(self, backends, read_selector=itemgetter(0),
runner='QueuedRunner', **kwargs):
assert backends, "you should provide at least one backend"
self.backends = []
for backend, backend_options in backends:
if isinstance(backend, six.string_types):
backend = import_string(backend)
self.backends.append(backend(**backend_options))
self.read_selector = read_selector
self.runner = {'QueuedRunner': QueuedRunner, 'ImmediateRunner': ImmediateRunner}[runner]()
super(TagStorage, self).__init__(**kwargs)
def test_process_pending_one_batch(self, cluster_get):
attachment_cache = import_string('sentry.attachments.redis.RbAttachmentCache')(hosts=[])
cluster_get.assert_any_call('SENTRY_CACHE_OPTIONS', {'hosts': []})
assert isinstance(attachment_cache.inner, RbCache)
assert attachment_cache.inner.client is CLIENT
rv = attachment_cache.get('foo')
assert len(rv) == 1
attachment = rv[0]
assert attachment.meta() == {
'type': 'event.attachment',
'name': 'foo.txt',
'content_type': 'text/plain'
}
assert attachment.data == b'Hello World!'
def init_registry() -> RuleRegistry:
from sentry.constants import _SENTRY_RULES
from sentry.plugins.base import plugins
from sentry.utils.imports import import_string
from sentry.utils.safe import safe_execute
registry = RuleRegistry()
for rule in _SENTRY_RULES:
cls = import_string(rule)
registry.add(cls)
for plugin in plugins.all(version=2):
for cls in safe_execute(plugin.get_rules, _with_transaction=False) or ():
registry.add(cls)
return registry
def init_registry():
from sentry.constants import SENTRY_RULES
from sentry.plugins import plugins
from sentry.utils.imports import import_string
from sentry.utils.safe import safe_execute
registry = RuleRegistry()
for rule in SENTRY_RULES:
cls = import_string(rule)
registry.add(cls)
for plugin in plugins.all(version=2):
for cls in (safe_execute(plugin.get_rules) or ()):
register.add(cls)
return registry
def register_plugins(settings, raise_on_plugin_load_failure=False):
from pkg_resources import iter_entry_points
from sentry.plugins.base import plugins
# entry_points={
# 'sentry.plugins': [
# 'phabricator = sentry_phabricator.plugins:PhabricatorPlugin'
# ],
# },
for ep in iter_entry_points("sentry.plugins"):
try:
plugin = ep.load()
except Exception:
import traceback
click.echo(
f"Failed to load plugin {ep.name!r}:\n{traceback.format_exc()}",
err=True)
if raise_on_plugin_load_failure:
raise
else:
plugins.register(plugin)
for plugin in plugins.all(version=None):
init_plugin(plugin)
from sentry import integrations
from sentry.utils.imports import import_string
for integration_path in settings.SENTRY_DEFAULT_INTEGRATIONS:
try:
integration_cls = import_string(integration_path)
except Exception:
import traceback
click.echo(
f"Failed to load integration {integration_path!r}:\n{traceback.format_exc()}",
err=True,
)
else:
integrations.register(integration_cls)
for integration in integrations.all():
try:
integration.setup()
except AttributeError:
pass
def _process_single_incr(self, key):
client = self.cluster.get_routing_client()
lock_key = self._make_lock_key(key)
# prevent a stampede due to the way we use celery etas + duplicate
# tasks
if not client.set(lock_key, '1', nx=True, ex=10):
metrics.incr('buffer.revoked', tags={'reason': 'locked'})
self.logger.debug('buffer.revoked.locked', extra={'redis_key': key})
return
pending_key = self._make_pending_key_from_key(key)
try:
conn = self.cluster.get_local_client_for_key(key)
pipe = conn.pipeline()
pipe.hgetall(key)
pipe.zrem(pending_key, key)
pipe.delete(key)
values = pipe.execute()[0]
if not values:
metrics.incr('buffer.revoked', tags={'reason': 'empty'})
self.logger.debug('buffer.revoked.empty', extra={'redis_key': key})
return
model = import_string(values.pop('m'))
if values['f'].startswith('{'):
filters = self._load_values(json.loads(values.pop('f')))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
filters = pickle.loads(values.pop('f'))
incr_values = {}
extra_values = {}
for k, v in six.iteritems(values):
if k.startswith('i+'):
incr_values[k[2:]] = int(v)
elif k.startswith('e+'):
if v.startswith('['):
extra_values[k[2:]] = self._load_value(json.loads(v))
else:
# TODO(dcramer): legacy pickle support - remove in Sentry 9.1
extra_values[k[2:]] = pickle.loads(v)
super(RedisBuffer, self).process(model, incr_values, filters, extra_values)
finally:
client.delete(lock_key)
def validate_data(project, data, client=None):
ensure_valid_project_id(project, data)
if not data.get('message'):
data['message'] = '<no message value>'
if not data.get('event_id'):
data['event_id'] = uuid.uuid4().hex
elif len(data['event_id']) > 32:
raise InvalidData('Invalid value for \'event_id\': must be a 32 character identifier')
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp:
# Log the error, remove the timestamp, and continue
logger.error('Client %r passed an invalid value for timestamp %r',
client or '<unknown client>',
data['timestamp'],
)
del data['timestamp']
if data.get('modules') and type(data['modules']) != dict:
raise InvalidData('Invalid type for \'modules\': must be a mapping')
for k, v in data.items():
if k in RESERVED_FIELDS:
continue
if '.' not in k:
logger.error('Ignoring unknown attribute %r passed by client %r',
k,
client or '<unknown client>',
)
del data[k]
continue
try:
interface = import_string(k)
except (ImportError, AttributeError), e:
raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))
try:
data[k] = interface(**v).serialize()
except Exception, e:
raise InvalidData('Unable to validate interface, %r: %s' % (k, e))
def interfaces(self):
result = []
for key, data in self.data.iteritems():
if '.' not in key:
continue
try:
cls = import_string(key)
except ImportError:
continue # suppress invalid interfaces
value = safe_execute(cls, **data)
if not value:
continue
result.append((key, value))
return SortedDict((k, v) for k, v in sorted(result, key=lambda x: x[1].get_score(), reverse=True))
def register_plugins(settings):
from pkg_resources import iter_entry_points
from sentry.plugins import plugins
# entry_points={
# 'sentry.plugins': [
# 'phabricator = sentry_phabricator.plugins:PhabricatorPlugin'
# ],
# },
for ep in iter_entry_points('sentry.plugins'):
try:
plugin = ep.load()
except Exception:
import traceback
click.echo(
"Failed to load plugin %r:\n%s" % (ep.name, traceback.format_exc()),
err=True
)
else:
plugins.register(plugin)
for plugin in plugins.all(version=None):
init_plugin(plugin)
from sentry import integrations
from sentry.utils.imports import import_string
for integration_path in settings.SENTRY_DEFAULT_INTEGRATIONS:
try:
integration_cls = import_string(integration_path)
except Exception:
import traceback
click.echo(
"Failed to load integration %r:\n%s" % (integration_path, traceback.format_exc()),
err=True
)
else:
integrations.register(integration_cls)
for integration in integrations.all():
try:
integration.setup()
except AttributeError:
pass
def get_filters(project, request):
filter_list = []
for class_path in settings.SENTRY_FILTERS:
try:
filter_cls = import_string(class_path)
except Exception:
logger = logging.getLogger('sentry.errors')
logger.exception('Unable to import %s', class_path)
continue
filter_list.append(filter_cls(request, project))
for tag in project.get_tags():
filter_list.append(Filter(
column=tag,
label=TAG_LABELS.get(tag) or tag.replace('_', ' ').title(),
request=request,
project=project,
))
return filter_list
def validate_data(project, data, client=None):
for k in REQUIRED_FIELDS:
if not data.get(k):
raise InvalidData('Missing required parameter: %r' % k)
ensure_valid_project_id(project, data)
if 'event_id' not in data:
data['event_id'] = uuid.uuid4().hex
if 'timestamp' in data:
try:
process_data_timestamp(data)
except InvalidTimestamp:
# Log the error, remove the timestamp, and continue
logger.error('Client %r passed an invalid value for timestamp %r' % (
data['timestamp'],
client or '<unknown client>',
))
del data['timestamp']
if data.get('modules') and type(data['modules']) != dict:
raise InvalidData('Invalid type for \'modules\': must be a mapping')
for k, v in data.iteritems():
if k in RESERVED_FIELDS:
continue
if '.' not in k:
raise InvalidInterface('%r is not a valid interface name' % k)
try:
interface = import_string(k)
except (ImportError, AttributeError), e:
raise InvalidInterface('%r is not a valid interface name: %s' % (k, e))
try:
data[k] = interface(**v).serialize()
except Exception, e:
raise InvalidData('Unable to validate interface, %r: %s' % (k, e))
def validate_data(project, data, client=None):
for k in REQUIRED_FIELDS:
if not data.get(k):
raise InvalidData("Missing required parameter: %r" % k)
ensure_valid_project_id(project, data)
if "event_id" not in data:
data["event_id"] = uuid.uuid4().hex
if "timestamp" in data:
try:
process_data_timestamp(data)
except InvalidTimestamp:
# Log the error, remove the timestamp, and continue
logger.error(
"Client %r passed an invalid value for timestamp %r" % (data["timestamp"], client or "<unknown client>")
)
del data["timestamp"]
if data.get("modules") and type(data["modules"]) != dict:
raise InvalidData("Invalid type for 'modules': must be a mapping")
for k, v in data.iteritems():
if k in RESERVED_FIELDS:
continue
if "." not in k:
raise InvalidInterface("%r is not a valid interface name" % k)
try:
interface = import_string(k)
except (ImportError, AttributeError), e:
raise InvalidInterface("%r is not a valid interface name: %s" % (k, e))
try:
data[k] = interface(**v).serialize()
except Exception, e:
raise InvalidData("Unable to validate interface, %r: %s" % (k, e))
def get_instance(path, options):
cls = import_string(path)
return cls(**options)
'Ignored empty interface value: %s', k,
**client_metadata(client, project))
del data[k]
continue
import_path = INTERFACE_ALIASES.get(k, k)
if '.' not in import_path:
logger.info(
'Ignored unknown attribute: %s', k,
**client_metadata(client, project))
del data[k]
continue
try:
interface = import_string(import_path)
except (ImportError, AttributeError) as e:
logger.info(
'Invalid unknown attribute: %s', k,
**client_metadata(client, project, exception=e))
del data[k]
continue
value = data.pop(k)
try:
# HACK: exception allows you to pass the value as a list
# so let's try to actually support that
if isinstance(value, dict):
inst = interface(**value)
else:
inst = interface(value)
"""Sentry is cross-platform crash reporting built with love.
The configuration file is looked up in the `~/.sentry` config
directory but this can be overridden with the `SENTRY_CONF`
environment variable or be explicitly provided through the
`--config` parameter.
"""
# Elevate --config option to SENTRY_CONF env var, and just assume this
# always will exist down the line
if config:
os.environ['SENTRY_CONF'] = config
os.environ.setdefault('SENTRY_CONF', '~/.sentry')
# TODO(mattrobenolt): Autodiscover commands?
list(map(lambda cmd: cli.add_command(import_string(cmd)), (
'sentry.runner.commands.backup.export',
'sentry.runner.commands.backup.import_',
'sentry.runner.commands.cleanup.cleanup',
'sentry.runner.commands.config.config',
'sentry.runner.commands.createuser.createuser',
'sentry.runner.commands.devserver.devserver',
'sentry.runner.commands.django.django',
'sentry.runner.commands.exec.exec_',
'sentry.runner.commands.files.files',
'sentry.runner.commands.help.help',
'sentry.runner.commands.init.init',
'sentry.runner.commands.plugins.plugins',
'sentry.runner.commands.queues.queues',
'sentry.runner.commands.repair.repair',
'sentry.runner.commands.run.run',
