def finalize(self):
"""
Call before you generate a dataset
This caches all the guids in the db at time of calling to a list
creating a data structure which allows for fast access by the dataset object
and fixing the length of the the dataset
"""
lockname = self.key('lock')
lk = Lock(self.redis, lockname)
#logger.debug(f'getting lock {lockname}')
lk.acquire()
#logger.debug(f'getting lock {lockname}')
self.redis.set(self.key('finalized'), 'FINALIZED')
if lk.owned():
lk.release()
#logger.debug(f'released lock {lockname}')
self.episodes = []
self.episode_len = []
for episode in range(self.num_episodes()):
self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode())
self.episodes = sorted(self.episodes)
for episode_id in self.episodes:
self.episode_len.append(len(Episode(self, self.redis, episode_id)))
self.episode_off = []
offset = 0
for l in self.episode_len:
self.episode_off.append(offset)
offset += l
self.finalized = True
def test_redis_raise_or_lock_locked_and_expired(redis, backend):
lock = RedisLock(redis, "test", timeout=1)
lock.acquire()
time.sleep(1) # wait for lock to expire
backend.raise_or_lock(key="test", timeout=60)
assert redis.get("test") is not None
def test_redis_raise_or_lock_locked(redis, backend):
# Set to expire in 30 seconds!
lock = RedisLock(redis, "test", timeout=30)
lock.acquire()
with pytest.raises(AlreadyQueued) as e:
backend.raise_or_lock(key="test", timeout=60)
assert e.value.countdown == 30.0
assert e.value.message == "Expires in 30.0 seconds"
def test_apply_async_expired(redis):
lock = RedisLock(redis, "qo_example_a-1", timeout=1)
lock.acquire()
assert redis.get("qo_example_a-1") is not None
time.sleep(1)
example.apply_async(args=(1, ))
assert redis.get("qo_example_a-1") is None
def test_delay_expired(redis):
lock = RedisLock(redis, "qo_example_a-1", timeout=1)
lock.acquire()
assert redis.get("qo_example_a-1") is not None
time.sleep(1)
example.delay(1)
assert redis.get("qo_example_a-1") is None
def end(self):
if self.p is not None:
self.p.execute()
lockname = self.rollout.key('lock')
lk = Lock(self.redis, lockname)
#logger.debug(f'getting lock {lockname}')
lk.acquire()
#logger.debug(f'got lock {lockname}')
if not self.redis.exists(self.rollout.key('finalized')):
self.redis.lpush(self.rollout.key('episodes'), self.id)
self.redis.incrby(self.rollout.key('steps'), len(self))
if lk.owned():
lk.release()
class LockManager():
def __init__(self, lock_name, lock_timeout=10):
self.lock_name = lock_name
self.lock_timeout = lock_timeout
self.is_lock_free = False
def __enter__(self):
self.lock = Lock(
redis_client,
self.lock_name,
blocking_timeout=1,
timeout=self.lock_timeout,
)
try:
self.is_lock_free = self.lock.acquire(blocking=False)
except LockError:
log.error(
'lock acquire error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
return self
def __exit__(self, type, value, traceback):
if self.is_lock_free:
try:
self.lock.release()
except LockError:
log.error(
'lock release error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
def acquire(self, lox_name, lock_id, expires_seconds=None):
key = self.key(lox_name, lock_id)
lock = Lock(self.connection, key, timeout=expires_seconds)
# retry logic is handled in core.lock, so no blocking here
if lock.acquire(blocking=False):
return BackendLock(key, lox_name, lock_id, provider_lock=lock)
else:
raise LockInUseException("Lock {} has been acquired previously, possibly by another thread/process, and is not available.".format(key))
def acquire(self, lox_name, lock_id, expires_seconds=None):
key = self.key(lox_name, lock_id)
lock = Lock(self.connection, key, timeout=expires_seconds)
# retry logic is handled in core.lock, so no blocking here
if lock.acquire(blocking=False):
return BackendLock(key, lox_name, lock_id, provider_lock=lock)
else:
raise LockInUseException(
"Lock {} has been acquired previously, possibly by another thread/process, and is not available."
.format(key))
class RedisBackendLock(DbBackendLockAbc):
"""
A class providing an implementation of database backend lock of Shared Data Layer (SDL), when
backend database solution is Redis.
Args:
ns (str): Namespace under which this lock is targeted.
name (str): Lock name, identifies the lock key in a Redis database backend.
expiration (int, float): Lock expiration time after which the lock is removed if it hasn't
been released earlier by a 'release' method.
redis_backend (RedisBackend): Database backend object containing connection to Redis
database.
"""
lua_get_validity_time = None
# KEYS[1] - lock name
# ARGS[1] - token
# return < 0 in case of failure, otherwise return lock validity time in milliseconds.
LUA_GET_VALIDITY_TIME_SCRIPT = """
local token = redis.call('get', KEYS[1])
if not token then
return -10
end
if token ~= ARGV[1] then
return -11
end
return redis.call('pttl', KEYS[1])
"""
def __init__(self, ns: str, name: str, expiration: Union[int, float],
redis_backend: RedisBackend) -> None:
super().__init__(ns, name)
self.__redis = redis_backend.get_redis_connection(ns)
with _map_to_sdl_exception():
redis_lockname = '{' + ns + '},' + self._lock_name
self.__redis_lock = Lock(redis=self.__redis,
name=redis_lockname,
timeout=expiration)
self._register_scripts()
def __str__(self):
return str({
"lock DB type": "Redis",
"lock namespace": self._ns,
"lock name": self._lock_name,
"lock status": self._lock_status_to_string()
})
def acquire(self,
retry_interval: Union[int, float] = 0.1,
retry_timeout: Union[int, float] = 10) -> bool:
succeeded = False
self.__redis_lock.sleep = retry_interval
with _map_to_sdl_exception():
succeeded = self.__redis_lock.acquire(
blocking_timeout=retry_timeout)
return succeeded
def release(self) -> None:
with _map_to_sdl_exception():
self.__redis_lock.release()
def refresh(self) -> None:
with _map_to_sdl_exception():
self.__redis_lock.reacquire()
def get_validity_time(self) -> Union[int, float]:
validity = 0
if self.__redis_lock.local.token is None:
msg = u'Cannot get validity time of an unlocked lock %s' % self._lock_name
raise RejectedByBackend(msg)
with _map_to_sdl_exception():
validity = self.lua_get_validity_time(
keys=[self.__redis_lock.name],
args=[self.__redis_lock.local.token],
client=self.__redis)
if validity < 0:
msg = (
u'Getting validity time of a lock %s failed with error code: %d'
% (self._lock_name, validity))
raise RejectedByBackend(msg)
ftime = validity / 1000.0
if ftime.is_integer():
return int(ftime)
return ftime
def _register_scripts(self):
cls = self.__class__
client = self.__redis
if cls.lua_get_validity_time is None:
cls.lua_get_validity_time = client.register_script(
cls.LUA_GET_VALIDITY_TIME_SCRIPT)
def _lock_status_to_string(self) -> str:
try:
if self.__redis_lock.locked():
if self.__redis_lock.owned():
return 'locked'
return 'locked by someone else'
return 'unlocked'
except (redis_exceptions.RedisError) as exc:
return f'Error: {str(exc)}'
def testRedisNativeLock(db):
l = Lock(db.redis, 'zelock', timeout=3.0)
l.acquire()
l.release()