"""Check if the specific `id` has at least `amount` tokens, and decrease the bucket if it is the case
:param id: an id identifying the entity to test, must be convertible to a string
:param amount: an integer or float value
:return: whether the entity has the necessary tokens
"""
key_prefix = f"TokenBucket:{self.name}:{str(id)}:"
res = token_bucket_script(
keys=[f"{key_prefix}v", f"{key_prefix}t"],
args=[self.max, self.interval,
get_current_timestamp(), amount],
client=get_redis_client(),
)
return bool(res)
def reset(self, id):
key_prefix = f"TokenBucket:{self.name}:{str(id)}:"
get_redis_client().pipeline().delete(f"{key_prefix}v").delete(
f"{key_prefix}t").execute()
with open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"token_bucket.lua"),
mode="rb",
) as f:
token_bucket_script = Script(None, f.read())
import six
from django.utils import timezone
from pkg_resources import resource_string
from redis.client import Script
from sentry.tsdb.base import BaseTSDB
from sentry.utils.dates import to_timestamp
from sentry.utils.redis import check_cluster_versions, get_cluster_from_options
from sentry.utils.versioning import Version
logger = logging.getLogger(__name__)
SketchParameters = namedtuple('SketchParameters', 'depth width capacity')
CountMinScript = Script(
None,
resource_string('sentry', 'scripts/tsdb/cmsketch.lua'),
)
class RedisTSDB(BaseTSDB):
"""
A time series storage backend for Redis.
The time series API supports three data types:
* simple counters
* distinct counters (number of unique elements seen)
* frequency tables (a set of items ranked by most frequently observed)
The backend also supports virtual nodes (``vnodes``) which controls shard
distribution. This value should be set to the anticipated maximum number of
# Trims a timeline to a maximum number of records.
# Returns the number of keys that were deleted.
# KEYS: {TIMELINE}
# ARGV: {LIMIT}
TRUNCATE_TIMELINE_SCRIPT = """\
local keys = redis.call('ZREVRANGE', KEYS[1], ARGV[1], -1)
for i, record in pairs(keys) do
redis.call('DEL', KEYS[1] .. ':{TIMELINE_RECORD_PATH_COMPONENT}:' .. record)
redis.call('ZREM', KEYS[1], record)
end
return table.getn(keys)
""".format(TIMELINE_RECORD_PATH_COMPONENT=TIMELINE_RECORD_PATH_COMPONENT)
# XXX: Passing `None` as the first argument is a dirty hack to allow us to use
# this more easily with the cluster
ensure_timeline_scheduled = Script(None, ENSURE_TIMELINE_SCHEDULED_SCRIPT)
truncate_timeline = Script(None, TRUNCATE_TIMELINE_SCRIPT)
class RedisBackend(Backend):
"""
Implements the digest backend API, backed by Redis.
Each timeline is modeled as a sorted set, and also maintains a separate key
that contains the last time the digest was processed (used for scheduling.)
.. code::
redis:6379> ZREVRANGEBYSCORE "d:t:mail:p:1" inf -inf WITHSCORES
1) "433be20b807c4cd49a132de69c0f6c55"
2) "1444847625"
from pkg_resources import resource_string
from redis.client import Script
from sentry.tsdb.base import BaseTSDB
from sentry.utils.dates import to_datetime, to_timestamp
from sentry.utils.redis import check_cluster_versions, get_cluster_from_options
from sentry.utils.versioning import Version
from six.moves import reduce
from sentry.utils.compat import map
from sentry.utils.compat import zip
logger = logging.getLogger(__name__)
SketchParameters = namedtuple("SketchParameters", "depth width capacity")
CountMinScript = Script(None,
resource_string("sentry", "scripts/tsdb/cmsketch.lua"))
class SuppressionWrapper(object):
"""\
Wraps a context manager and prevents any exceptions raised either during
the managed block or the exiting of the wrapped manager from propagating.
You probably shouldn't use this.
"""
def __init__(self, wrapped):
self.wrapped = wrapped
def __enter__(self):
return self.wrapped.__enter__()
# ARGV[1] = id # noqa: E800
# ARGV[2] = semaphore limit # noqa: E800
# ARGV[3] = expiry in seconds # noqa: E800
# KEY[1] = lock key # noqa: E800
# KEY[2] = signal key # noqa: E800
ACQUIRE_LOCK_SCRIPT = Script(
None,
b"""
redis.replicate_commands()
local expire_score = tonumber(redis.call("TIME")[1])
local purged_key_count = redis.call("zremrangebyscore", KEYS[1], '-inf', expire_score)
redis.call("del", KEYS[2])
for i=1,purged_key_count do redis.call("lpush", KEYS[2], 1) end
redis.call("pexpire", KEYS[2], 1000)
if redis.call("zcount", KEYS[1], '-inf', 'inf') < tonumber(ARGV[2]) then
redis.call("zadd", KEYS[1], expire_score + tonumber(ARGV[3]), ARGV[1])
return 1
else
return 0
end
""",
)
# ARGV[1] = id # noqa: E800
# ARGV[2] = expiry in seconds # noqa: E800
# KEY[1] = lock key # noqa: E800
EXTEND_LOCK_SCRIPT = Script(
None,