def test_redis_raise_or_lock_locked_and_expired(redis, backend):
lock = RedisLock(redis, "test", timeout=1)
lock.acquire()
time.sleep(1) # wait for lock to expire
backend.raise_or_lock(key="test", timeout=60)
assert redis.get("test") is not None
class LockManager():
def __init__(self, lock_name, lock_timeout=10):
self.lock_name = lock_name
self.lock_timeout = lock_timeout
self.is_lock_free = False
def __enter__(self):
self.lock = Lock(
redis_client,
self.lock_name,
blocking_timeout=1,
timeout=self.lock_timeout,
)
try:
self.is_lock_free = self.lock.acquire(blocking=False)
except LockError:
log.error(
'lock acquire error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
return self
def __exit__(self, type, value, traceback):
if self.is_lock_free:
try:
self.lock.release()
except LockError:
log.error(
'lock release error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
def acquire(self, lox_name, lock_id, expires_seconds=None):
key = self.key(lox_name, lock_id)
lock = Lock(self.connection, key, timeout=expires_seconds)
# retry logic is handled in core.lock, so no blocking here
if lock.acquire(blocking=False):
return BackendLock(key, lox_name, lock_id, provider_lock=lock)
else:
raise LockInUseException("Lock {} has been acquired previously, possibly by another thread/process, and is not available.".format(key))
def test_delay_expired(redis):
lock = RedisLock(redis, "qo_example_a-1", timeout=1)
lock.acquire()
assert redis.get("qo_example_a-1") is not None
time.sleep(1)
example.delay(1)
assert redis.get("qo_example_a-1") is None
def test_redis_raise_or_lock_locked(redis, backend):
# Set to expire in 30 seconds!
lock = RedisLock(redis, "test", timeout=30)
lock.acquire()
with pytest.raises(AlreadyQueued) as e:
backend.raise_or_lock(key="test", timeout=60)
assert e.value.countdown == 30.0
assert e.value.message == "Expires in 30.0 seconds"
def acquire(self, lox_name, lock_id, expires_seconds=None):
key = self.key(lox_name, lock_id)
lock = Lock(self.connection, key, timeout=expires_seconds)
# retry logic is handled in core.lock, so no blocking here
if lock.acquire(blocking=False):
return BackendLock(key, lox_name, lock_id, provider_lock=lock)
else:
raise LockInUseException(
"Lock {} has been acquired previously, possibly by another thread/process, and is not available."
.format(key))
def test_apply_async_expired(redis):
lock = RedisLock(redis, "qo_example_a-1", timeout=1)
lock.acquire()
assert redis.get("qo_example_a-1") is not None
time.sleep(1)
example.apply_async(args=(1, ))
assert redis.get("qo_example_a-1") is None
def __init__(self, ns: str, name: str, expiration: Union[int, float],
redis_backend: RedisBackend) -> None:
super().__init__(ns, name)
self.__redis = redis_backend.get_redis_connection(ns)
with _map_to_sdl_exception():
redis_lockname = '{' + ns + '},' + self._lock_name
self.__redis_lock = Lock(redis=self.__redis,
name=redis_lockname,
timeout=expiration)
self._register_scripts()
def __enter__(self):
self.lock = Lock(
redis_client,
self.lock_name,
blocking_timeout=1,
timeout=self.lock_timeout,
)
try:
self.is_lock_free = self.lock.acquire(blocking=False)
except LockError:
log.error(
'lock acquire error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
return self
def aggregate_day_after_hour_aggregated(sender, left_boundary, right_boundary,
**kwargs):
aggregate_queryset = HourAggregate.objects.filter(
timestamp__gte=left_boundary, timestamp__lt=right_boundary)
aggregate_queryset.query.clear_ordering(force_empty=True)
aggregate_queryset = aggregate_queryset.values(
'admin_id', 'instance_id', 'instance_name',
'source').annotate(value=Sum('value'))
day_date = datetime(left_boundary.year,
left_boundary.month,
left_boundary.day,
tzinfo=pytz.utc)
with Lock(redis=redis,
name='day_aggregation_{}'.format(day_date.isoformat())):
for aggregate_dict in aggregate_queryset.iterator():
filter_data = {
'timestamp': day_date,
'instance_name': aggregate_dict.get('instance_name'),
'source': aggregate_dict.get('source'),
'admin_id': aggregate_dict.get('admin_id')
}
if not DayAggregate.objects.filter(**filter_data).update(
value=F('value') + aggregate_dict.get('value')):
DayAggregate.objects.create(timestamp=day_date,
**aggregate_dict)
def get_hash_cache(name, key, ex=None, none_callback=None, *args, **kwargs):
"""
name : redis hash name
key : redis 键名
ex : 超期时间 (单位：秒)
none_callback : 返回None值，调用回调函数
args : 回调参数
kwags : 回调参数
"""
cli = RedisPool().connection()
with Lock(cli, _dist_lock_key(name), timeout=20):
v = cli.hget(name, key)
if v is None and none_callback:
if DEBUG_CACHE:
print "no redis cache hit. %s." % key
v = none_callback(*args, **kwargs)
try:
nv = JSONDataEncoder.encode(v)
except:
nv = v
cli.hset(name, key, nv)
_set_expire_time(cli, name, ex)
else:
if DEBUG_CACHE:
print "cache redis hit. %s." % key
try:
v = JSONDataEncoder.decode(v)
except:
pass
return v
def get_hash_map_cache(name,
keys,
ex=None,
none_callback=None,
*args,
**kwargs):
"""
name : name
keys : key list
ex : 超期时间 (单位：秒)
none_callback : 返回None值，调用回调函数
args : 回调参数
kwags : 回调参数
"""
cli = RedisPool().connection()
with Lock(cli, _dist_lock_key(name), timeout=20):
v = cli.hmget(name, keys)
if v is None and none_callback:
if DEBUG_CACHE:
print "no redis cache hit. %s." % keys
v = none_callback(*args, **kwargs)
cli.hmset(name, v)
_set_expire_time(cli, name, ex)
if v and (isinstance(keys, str) or
(isinstance(keys, list) and len(keys) == 1)):
v = v[-1]
return v
def create_lock(cls, id, r_obj=None):
import config
from redis.lock import LuaLock as Lock
redis_key = 'lock_%s_%s' % (cls.__name__, id)
return Lock(r_obj or rds_tmp, redis_key,
sleep=config.REDIS_LOCK_SLEEP,
timeout=config.REDIS_LOCK_TIMEOUT)
def refresh_redis_cache(name, ex):
"""
name : name
ex : 超期时间 (单位：秒)
"""
cli = RedisPool().connection()
with Lock(cli, _dist_lock_key(name), timeout=20):
_set_expire_time(cli, name, ex)
def finalize(self):
"""
Call before you generate a dataset
This caches all the guids in the db at time of calling to a list
creating a data structure which allows for fast access by the dataset object
and fixing the length of the the dataset
"""
lockname = self.key('lock')
lk = Lock(self.redis, lockname)
#logger.debug(f'getting lock {lockname}')
lk.acquire()
#logger.debug(f'getting lock {lockname}')
self.redis.set(self.key('finalized'), 'FINALIZED')
if lk.owned():
lk.release()
#logger.debug(f'released lock {lockname}')
self.episodes = []
self.episode_len = []
for episode in range(self.num_episodes()):
self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode())
self.episodes = sorted(self.episodes)
for episode_id in self.episodes:
self.episode_len.append(len(Episode(self, self.redis, episode_id)))
self.episode_off = []
offset = 0
for l in self.episode_len:
self.episode_off.append(offset)
offset += l
self.finalized = True
def __init__(self, unique_id=uuid4(), redis_server=None, *args, **kwargs):
self._unique_id = unique_id
self._redis_server = redis_server or StrictRedis()
refresh_lock = Lock(redis=self._redis_server,
name='{0}_lock'.format(self._unique_id))
super(RedisManagedOAuth2Mixin,
self).__init__(*args, refresh_lock=refresh_lock, **kwargs)
if self._access_token is None:
self._get_and_update_current_tokens()
def redis_lock_from_event(event):
from django_redis import get_redis_connection
from redis.lock import Lock
if not hasattr(event, '_lock') or not event._lock:
rc = get_redis_connection("redis")
event._lock = Lock(redis=rc,
name='pretix_event_%s' % event.id,
timeout=LOCK_TIMEOUT)
return event._lock
def release_lock(lock_name, token):
"""
Release a lock
Args:
lock_name (str): The lock key in redis
token (bytes): The unique id used
Returns:
bool: True if the lock was successfully released
"""
# this is a StrictRedis instance, we need this for the script installation that LuaLock uses
redis = caches['redis'].client.get_client()
lock = LuaLock(redis, lock_name)
try:
lock.do_release(token)
except LockError:
# If the lock is expired we don't want to raise an error
pass
def set_hash_map_cache(name, mapping, ex=None):
"""
name : name
mapping : mapping dict
ex : 超期时间 (单位：秒)
"""
cli = RedisPool().connection()
with Lock(cli, _dist_lock_key(name), timeout=20):
if DEBUG_CACHE:
print "set redis hash map. %s %s" % (name, ex)
cli.hmset(name, mapping)
_set_expire_time(cli, name, ex)
return True
def greenlet_scheduler(self):
redis_scheduler_lock_key = "%s:schedulerlock" % get_current_config(
)["redis_prefix"]
while True:
with Lock(connections.redis,
redis_scheduler_lock_key,
timeout=self.config["scheduler_interval"] + 10,
blocking=False,
thread_local=False):
self.scheduler.check()
time.sleep(self.config["scheduler_interval"])
class RedisLock:
'''
An awaitable redis Lock. The event loop won't be blocked when acquiring the lock.
'''
def __init__(self, redi, name: str = "", sleep: int = 0.02, timeout:int = 5, blocking_timeout: int = 10, thread_local: bool = False):
self._lock = RLock(redi, name, sleep=sleep, timeout=timeout, blocking_timeout=blocking_timeout, thread_local=thread_local)
async def __aenter__(self):
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, self._lock.acquire)
return self
async def __aexit__(self, *args):
self._lock.release()
async def acquire(self):
'''Acquire the lock without locking the loop'''
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, self._lock.acquire)
def release(self):
'''Release the lock, this is not async for the sake of easier cleanup (e.g. registering `atexit`)'''
self._lock.release()
def wrapper(*args, **kwargs):
init_transaction()
with Lock(redis_lock, get_trans_lock_name(), timeout=5) as look:
try:
begin_transaction()
result = func(*args, **kwargs)
end_transaction()
close_transaction(get_transaction_id())
return result
except Exception as e:
rollback(get_transaction_id())
raise e
finally:
release_transaction()
def set_cache(key, value, ex=None):
"""
key : redis 键名
value : redis 键值
ex : 超期时间 (单位：秒)
"""
cli = RedisPool().connection()
with Lock(cli, _dist_lock_key(key), timeout=20):
if DEBUG_CACHE:
print "set redis cache. %s %s" % (key, ex)
try:
v = JSONDataEncoder.encode(value)
except:
v = value
cli.set(key, v, ex)
return True
def redis_lock(name: str, **kwargs) -> Generator:
"""
Acquire a Redis lock. This is a wrapper around redis.lock.Lock(), that also works in tests (there, the lock is
always granted without any checks).
Relevant kwargs are:
- blocking_timeout: how many seconds to try to acquire the lock. Use 0 for a non-blocking lock.
The default is None, which means we wait forever.
- timeout: how many seconds to keep the lock for. The default is None, which means it remains locked forever.
Raises redis.exceptions.LockError if the lock couldn't be acquired or released.
"""
if settings.DJTRIGGERS_REDIS_URL.startswith('redis'): # pragma: no cover
with Lock(redis=Redis.from_url(settings.DJTRIGGERS_REDIS_URL), name=name, **kwargs):
yield
else:
yield
def raise_or_lock(self, key, timeout):
"""
Checks if the task is locked and raises an exception, else locks
the task. By default, the tasks and the key expire after 60 minutes.
(meaning it will not be executed and the lock will clear).
"""
acquired = Lock(self.redis,
key,
timeout=timeout,
blocking=self.blocking,
blocking_timeout=self.blocking_timeout).acquire()
if not acquired:
# Time remaining in milliseconds
# https://redis.io/commands/pttl
ttl = self.redis.pttl(key)
raise AlreadyQueued(ttl / 1000.)
def end(self):
if self.p is not None:
self.p.execute()
lockname = self.rollout.key('lock')
lk = Lock(self.redis, lockname)
#logger.debug(f'getting lock {lockname}')
lk.acquire()
#logger.debug(f'got lock {lockname}')
if not self.redis.exists(self.rollout.key('finalized')):
self.redis.lpush(self.rollout.key('episodes'), self.id)
self.redis.incrby(self.rollout.key('steps'), len(self))
if lk.owned():
lk.release()
def set_hash_cache(name, key, value, ex=None):
"""
name : redis hash name
key : redis 键名
value : redis 键值
ex : 超期时间 (单位：秒)
"""
cli = RedisPool().connection()
with Lock(cli, _dist_lock_key(name), timeout=20):
if DEBUG_CACHE:
print "set redis hash cache. %s %s" % (key, ex)
try:
v = JSONDataEncoder.encode(value)
except:
v = value
cli.hset(name, key, v)
_set_expire_time(cli, name, ex)
return True
def __init__(self,
persist_mode=False,
key_prefix='',
min_cache_time=5,
force_cache_time=False,
base='store',
path='cache',
redis_host='localhost',
redis_port=6379,
redis_db=1,
redis_file=None):
self.__key_prefix = key_prefix
self.__cache_key = '{}:cache'.format(key_prefix)
self.__persist_mode = persist_mode
self.__min_cache_time = min_cache_time
self.__force_cache_time = force_cache_time
self.__base_path = base
self.__resource_cache = get_triple_store()
self._r = get_kv(persist_mode,
redis_host,
redis_port,
redis_db,
redis_file,
base=base,
path=path)
self.__lock = Lock(self._r, key_prefix)
self.__mlock = TLock()
self.__memory_graphs = {}
self.__memory_order = []
self.__resources_ts = {}
# Clean temporal folders under 'base' (others than 'path' subfolder)
for sub in filter(lambda x: x != path,
get_immediate_subdirectories(base)):
shutil.rmtree('{}/{}'.format(self.__base_path, sub))
for lock_key in self._r.keys('{}:l*'.format(self.__key_prefix)):
self._r.delete(lock_key)
self.__enabled = True
self.__purge_th = Thread(target=self.__purge)
self.__purge_th.daemon = True
self.__purge_th.start()
def __init__(self,
persist_mode=None,
key_prefix='',
min_cache_time=5,
force_cache_time=False,
base='store',
path='cache',
redis_host='localhost',
redis_port=6379,
redis_db=1,
redis_file=None,
graph_memory_limit=5000):
self.__key_prefix = key_prefix
self.__cache_key = '{}:cache'.format(key_prefix)
self.__persist_mode = persist_mode
self.__min_cache_time = min_cache_time
self.__force_cache_time = force_cache_time
self.__base_path = base
self._r = get_kv(persist_mode,
redis_host,
redis_port,
redis_db,
redis_file,
base=base,
path=path)
self.__lock = Lock(self._r, key_prefix)
self.__mlock = TLock()
self.__graph_memory_limit = graph_memory_limit
self.__memory_graphs = {}
self.__memory_order = []
self.__resources_ts = {}
for lock_key in self._r.keys('{}:l*'.format(self.__key_prefix)):
self._r.delete(lock_key)
self._r.delete(key_prefix)
self.__enabled = True
self.__purge_th = Thread(target=self.__purge)
self.__purge_th.daemon = True
self.__purge_th.start()
def __init__(self, unique_id=uuid4(), redis_server=None, *args, **kwargs):
"""
:param unique_id:
An identifier for this auth object. Auth instances which wish to share tokens must use the same ID.
:type unique_id:
`unicode`
:param redis_server:
An instance of a Redis server, configured to talk to Redis.
:type redis_server:
:class:`Redis`
"""
# pylint:disable=keyword-arg-before-vararg
self._unique_id = unique_id
self._redis_server = redis_server or StrictRedis()
refresh_lock = Lock(redis=self._redis_server,
name='{0}_lock'.format(self._unique_id))
super(RedisManagedOAuth2Mixin,
self).__init__(*args, refresh_lock=refresh_lock, **kwargs)
if self._access_token is None:
self._get_and_update_current_tokens()
def _send_smses(send_deferred=False, backend=None, limit=None):
# Get lock so there is only one sms sender at the same time.
if send_deferred:
send_lock_name = 'smsgateway_send_sms_deferred'
else:
send_lock_name = 'smsgateway_send_sms'
with Lock(redis=Redis.from_url(settings.SMSGATEWAY_REDIS_URL),
name='smsgateway-' + send_lock_name,
blocking_timeout=0):
successes, failures = 0, 0
try:
# Get SMSes that need to be sent (deferred or non-deferred)
if send_deferred:
to_send = QueuedSMS.objects.filter(priority=PRIORITY_DEFERRED)
else:
to_send = QueuedSMS.objects.exclude(priority=PRIORITY_DEFERRED)
if isinstance(limit, int):
to_send = to_send[:limit]
# Send each SMS
for sms in to_send:
if backend:
sms_using = backend
else:
sms_using = None if sms.using == '__none__' else sms.using
if send(sms.to, sms.content, sms.signature, sms_using,
sms.reliable):
# Successfully sent, remove from queue
sms.delete()
successes += 1
else:
# Failed to send, defer SMS
sms.defer()
failures += 1
finally:
if successes and failures:
statsd.gauge('smsgateway.success_rate', successes / failures)
else:
statsd.gauge('smsgateway.success_rate', 1)
