import operator
import struct
import time
from collections import Sequence
import mmh3
import six
from django.conf import settings
from sentry.utils import redis
from sentry.utils.datastructures import BidirectionalMapping
from sentry.utils.dates import to_timestamp
from sentry.utils.iterators import shingle
from sentry.utils.redis import load_script
index = load_script('similarity/index.lua')
logger = logging.getLogger('sentry.similarity')
class MinHashIndex(object):
def __init__(self, cluster, rows, bands, buckets, interval, retention):
self.cluster = cluster
self.rows = rows
sequence = itertools.count()
self.bands = [[next(sequence) for j in xrange(buckets)]
for i in xrange(bands)]
self.buckets = buckets
self.interval = interval
self.retention = retention
from __future__ import absolute_import
import six
from uuid import uuid4
from sentry.utils import redis
from sentry.utils.locking.backends import LockBackend
delete_lock = redis.load_script('utils/locking/delete_lock.lua')
class RedisLockBackend(LockBackend):
def __init__(self, cluster, prefix='l:', uuid=None):
if uuid is None:
uuid = uuid4().hex
self.cluster = cluster
self.prefix = prefix
self.uuid = uuid
def get_client(self, key, routing_key=None):
# This is a bit of an abstraction leak, but if an integer is provided
# we use that value to determine placement rather than the cluster
# router. This leaking allows us us to have more fine-grained control
# when data is already placed within partitions where the router
# wouldn't have placed it based on the key hash, and maintain data
# locality and failure isolation within those partitions. (For example,
# the entirety of a digest is bound to a specific partition by the
# *digest* key, even though a digest is composed of multiple values at
# different keys that would otherwise be placed on different
return '{0}:{1}:{2}'.format(namespace, TIMELINE_PATH_COMPONENT, key)
def make_last_processed_timestamp_key(timeline_key):
return '{0}:{1}'.format(timeline_key, TIMELINE_LAST_PROCESSED_TIMESTAMP_PATH_COMPONENT)
def make_digest_key(timeline_key):
return '{0}:{1}'.format(timeline_key, TIMELINE_DIGEST_PATH_COMPONENT)
def make_record_key(timeline_key, record):
return '{0}:{1}:{2}'.format(timeline_key, TIMELINE_RECORD_PATH_COMPONENT, record)
ensure_timeline_scheduled = load_script('digests/ensure_timeline_scheduled.lua')
truncate_timeline = load_script('digests/truncate_timeline.lua')
class RedisBackend(Backend):
"""
Implements the digest backend API, backed by Redis.
Each timeline is modeled as a sorted set, and also maintains a separate key
that contains the last time the digest was processed (used for scheduling.)
.. code::
redis:6379> ZREVRANGEBYSCORE "d:t:mail:p:1" inf -inf WITHSCORES
1) "433be20b807c4cd49a132de69c0f6c55"
2) "1444847625"
:license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import
from django.conf import settings
from time import time
from sentry.exceptions import InvalidConfiguration
from sentry.quotas.base import Quota, RateLimited, NotRateLimited
from sentry.utils.redis import (
load_script,
make_rb_cluster,
)
is_rate_limited = load_script('quotas/is_rate_limited.lua')
class RedisQuota(Quota):
#: The ``grace`` period allows accomodating for clock drift in TTL
#: calculation since the clock on the Redis instance used to store quota
#: metrics may not be in sync with the computer running this code.
grace = 60
def __init__(self, **options):
if not options:
# inherit default options from REDIS_OPTIONS
options = settings.SENTRY_REDIS_OPTIONS
super(RedisQuota, self).__init__(**options)
options.setdefault('hosts', {0: {}})
self.cluster = make_rb_cluster(options['hosts'])
import six
from time import time
from sentry import features
from sentry.constants import DataCategory
from sentry.quotas.base import NotRateLimited, Quota, QuotaConfig, QuotaScope, RateLimited
from sentry.utils.redis import (
get_dynamic_cluster_from_options,
validate_dynamic_cluster,
load_script,
)
from sentry.utils.compat import map
from sentry.utils.compat import zip
is_rate_limited = load_script("quotas/is_rate_limited.lua")
class RedisQuota(Quota):
#: The ``grace`` period allows accommodating for clock drift in TTL
#: calculation since the clock on the Redis instance used to store quota
#: metrics may not be in sync with the computer running this code.
grace = 60
def __init__(self, **options):
self.is_redis_cluster, self.cluster, options = get_dynamic_cluster_from_options(
"SENTRY_QUOTA_OPTIONS", options)
# Based on the `is_redis_cluster` flag, self.cluster is set two one of
# the following two objects:
# - false: `cluster` is a `RBCluster`. Call `get_local_client_for_key`
from __future__ import absolute_import
import itertools
import time
from sentry.similarity.backends.abstract import AbstractIndexBackend
from sentry.utils.iterators import chunked
from sentry.utils.redis import load_script
index = load_script("similarity/index.lua")
def band(n, value):
assert len(value) % n == 0
return list(chunked(value, len(value) / n))
def flatten(value):
return list(itertools.chain.from_iterable(value))
class RedisScriptMinHashIndexBackend(AbstractIndexBackend):
def __init__(self, cluster, namespace, signature_builder, bands, interval,
retention, candidate_set_limit):
self.cluster = cluster
self.namespace = namespace
self.signature_builder = signature_builder
self.bands = bands
self.interval = interval
self.retention = retention
self.candidate_set_limit = candidate_set_limit
import operator
import struct
import time
from collections import Sequence
import mmh3
import six
from django.conf import settings
from sentry.utils import redis
from sentry.utils.datastructures import BidirectionalMapping
from sentry.utils.dates import to_timestamp
from sentry.utils.iterators import shingle
from sentry.utils.redis import load_script
index = load_script('similarity/index.lua')
logger = logging.getLogger('sentry.similarity')
class MinHashIndex(object):
def __init__(self, cluster, rows, bands, buckets, interval, retention):
self.cluster = cluster
self.rows = rows
sequence = itertools.count()
self.bands = [[next(sequence) for j in xrange(buckets)] for i in xrange(bands)]
self.buckets = buckets
self.interval = interval
self.retention = retention
import logging
from dataclasses import dataclass
from time import time
from django.conf import settings
from sentry.exceptions import InvalidConfiguration
from sentry.utils import redis
logger = logging.getLogger(__name__)
ErrorLimit = float("inf")
DEFAULT_MAX_TTL_SECONDS = 30
rate_limit_info = redis.load_script("ratelimits/api_limiter.lua")
@dataclass
class ConcurrentLimitInfo:
limit: int
current_executions: int
limit_exceeded: bool
class ConcurrentRateLimiter:
def __init__(self, max_tll_seconds: int = DEFAULT_MAX_TTL_SECONDS) -> None:
cluster_key = getattr(settings, "SENTRY_RATE_LIMIT_REDIS_CLUSTER",
"default")
self.client = redis.redis_clusters.get(cluster_key)
self.max_ttl_seconds = max_tll_seconds
import time
from contextlib import contextmanager
from redis.client import ResponseError
from sentry.digests import Record, ScheduleEntry
from sentry.digests.backends.base import Backend, InvalidState
from sentry.utils.locking.backends.redis import RedisLockBackend
from sentry.utils.locking.manager import LockManager
from sentry.utils.redis import (check_cluster_versions,
get_cluster_from_options, load_script)
from sentry.utils.versioning import Version
logger = logging.getLogger('sentry.digests')
script = load_script('digests/digests.lua')
class RedisBackend(Backend):
"""
Implements the digest backend API, backed by Redis.
Each timeline is modeled as a sorted set, and also maintains a separate key
that contains the last time the digest was processed (used for scheduling.)
.. code::
redis:6379> ZREVRANGEBYSCORE "d:t:mail:p:1" inf -inf WITHSCORES
1) "433be20b807c4cd49a132de69c0f6c55"
2) "1444847625"
3) "0f9d5fe4b5b3400fab85d9a841aa8467"
import six
import time
from contextlib import contextmanager
from redis.client import ResponseError
from sentry.digests import Record, ScheduleEntry
from sentry.digests.backends.base import Backend, InvalidState
from sentry.utils.locking.backends.redis import RedisLockBackend
from sentry.utils.locking.manager import LockManager
from sentry.utils.redis import (check_cluster_versions, get_cluster_from_options, load_script)
from sentry.utils.versioning import Version
logger = logging.getLogger('sentry.digests')
script = load_script('digests/digests.lua')
class RedisBackend(Backend):
"""
Implements the digest backend API, backed by Redis.
Each timeline is modeled as a sorted set, and also maintains a separate key
that contains the last time the digest was processed (used for scheduling.)
.. code::
redis:6379> ZREVRANGEBYSCORE "d:t:mail:p:1" inf -inf WITHSCORES
1) "433be20b807c4cd49a132de69c0f6c55"
2) "1444847625"
3) "0f9d5fe4b5b3400fab85d9a841aa8467"
from rb.clients import LocalClient
from redis.exceptions import ResponseError
from sentry.digests import Record, ScheduleEntry
from sentry.digests.backends.base import Backend, InvalidState
from sentry.utils.compat import map
from sentry.utils.locking.backends.redis import RedisLockBackend
from sentry.utils.locking.lock import Lock
from sentry.utils.locking.manager import LockManager
from sentry.utils.redis import check_cluster_versions, get_cluster_from_options, load_script
from sentry.utils.versioning import Version
logger = logging.getLogger("sentry.digests")
script = load_script("digests/digests.lua")
class RedisBackend(Backend):
"""
Implements the digest backend API, backed by Redis.
Each timeline is modeled as a sorted set, and also maintains a separate key
that contains the last time the digest was processed (used for scheduling.)
.. code::
redis:6379> ZREVRANGEBYSCORE "d:t:mail:p:1" inf -inf WITHSCORES
1) "433be20b807c4cd49a132de69c0f6c55"
2) "1444847625"
3) "0f9d5fe4b5b3400fab85d9a841aa8467"
