def finalize(self):
"""
Call before you generate a dataset
This caches all the guids in the db at time of calling to a list
creating a data structure which allows for fast access by the dataset object
and fixing the length of the the dataset
"""
lockname = self.key('lock')
lk = Lock(self.redis, lockname)
#logger.debug(f'getting lock {lockname}')
lk.acquire()
#logger.debug(f'getting lock {lockname}')
self.redis.set(self.key('finalized'), 'FINALIZED')
if lk.owned():
lk.release()
#logger.debug(f'released lock {lockname}')
self.episodes = []
self.episode_len = []
for episode in range(self.num_episodes()):
self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode())
self.episodes = sorted(self.episodes)
for episode_id in self.episodes:
self.episode_len.append(len(Episode(self, self.redis, episode_id)))
self.episode_off = []
offset = 0
for l in self.episode_len:
self.episode_off.append(offset)
offset += l
self.finalized = True
class LockManager():
def __init__(self, lock_name, lock_timeout=10):
self.lock_name = lock_name
self.lock_timeout = lock_timeout
self.is_lock_free = False
def __enter__(self):
self.lock = Lock(
redis_client,
self.lock_name,
blocking_timeout=1,
timeout=self.lock_timeout,
)
try:
self.is_lock_free = self.lock.acquire(blocking=False)
except LockError:
log.error(
'lock acquire error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
return self
def __exit__(self, type, value, traceback):
if self.is_lock_free:
try:
self.lock.release()
except LockError:
log.error(
'lock release error',
extra={'data': {'lock_name': self.lock_name}},
exc_info=True,
)
def end(self):
if self.p is not None:
self.p.execute()
lockname = self.rollout.key('lock')
lk = Lock(self.redis, lockname)
#logger.debug(f'getting lock {lockname}')
lk.acquire()
#logger.debug(f'got lock {lockname}')
if not self.redis.exists(self.rollout.key('finalized')):
self.redis.lpush(self.rollout.key('episodes'), self.id)
self.redis.incrby(self.rollout.key('steps'), len(self))
if lk.owned():
lk.release()
class RedisLock:
'''
An awaitable redis Lock. The event loop won't be blocked when acquiring the lock.
'''
def __init__(self, redi, name: str = "", sleep: int = 0.02, timeout:int = 5, blocking_timeout: int = 10, thread_local: bool = False):
self._lock = RLock(redi, name, sleep=sleep, timeout=timeout, blocking_timeout=blocking_timeout, thread_local=thread_local)
async def __aenter__(self):
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, self._lock.acquire)
return self
async def __aexit__(self, *args):
self._lock.release()
async def acquire(self):
'''Acquire the lock without locking the loop'''
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, self._lock.acquire)
def release(self):
'''Release the lock, this is not async for the sake of easier cleanup (e.g. registering `atexit`)'''
self._lock.release()
class RedisBackendLock(DbBackendLockAbc):
"""
A class providing an implementation of database backend lock of Shared Data Layer (SDL), when
backend database solution is Redis.
Args:
ns (str): Namespace under which this lock is targeted.
name (str): Lock name, identifies the lock key in a Redis database backend.
expiration (int, float): Lock expiration time after which the lock is removed if it hasn't
been released earlier by a 'release' method.
redis_backend (RedisBackend): Database backend object containing connection to Redis
database.
"""
lua_get_validity_time = None
# KEYS[1] - lock name
# ARGS[1] - token
# return < 0 in case of failure, otherwise return lock validity time in milliseconds.
LUA_GET_VALIDITY_TIME_SCRIPT = """
local token = redis.call('get', KEYS[1])
if not token then
return -10
end
if token ~= ARGV[1] then
return -11
end
return redis.call('pttl', KEYS[1])
"""
def __init__(self, ns: str, name: str, expiration: Union[int, float],
redis_backend: RedisBackend) -> None:
super().__init__(ns, name)
self.__redis = redis_backend.get_redis_connection(ns)
with _map_to_sdl_exception():
redis_lockname = '{' + ns + '},' + self._lock_name
self.__redis_lock = Lock(redis=self.__redis,
name=redis_lockname,
timeout=expiration)
self._register_scripts()
def __str__(self):
return str({
"lock DB type": "Redis",
"lock namespace": self._ns,
"lock name": self._lock_name,
"lock status": self._lock_status_to_string()
})
def acquire(self,
retry_interval: Union[int, float] = 0.1,
retry_timeout: Union[int, float] = 10) -> bool:
succeeded = False
self.__redis_lock.sleep = retry_interval
with _map_to_sdl_exception():
succeeded = self.__redis_lock.acquire(
blocking_timeout=retry_timeout)
return succeeded
def release(self) -> None:
with _map_to_sdl_exception():
self.__redis_lock.release()
def refresh(self) -> None:
with _map_to_sdl_exception():
self.__redis_lock.reacquire()
def get_validity_time(self) -> Union[int, float]:
validity = 0
if self.__redis_lock.local.token is None:
msg = u'Cannot get validity time of an unlocked lock %s' % self._lock_name
raise RejectedByBackend(msg)
with _map_to_sdl_exception():
validity = self.lua_get_validity_time(
keys=[self.__redis_lock.name],
args=[self.__redis_lock.local.token],
client=self.__redis)
if validity < 0:
msg = (
u'Getting validity time of a lock %s failed with error code: %d'
% (self._lock_name, validity))
raise RejectedByBackend(msg)
ftime = validity / 1000.0
if ftime.is_integer():
return int(ftime)
return ftime
def _register_scripts(self):
cls = self.__class__
client = self.__redis
if cls.lua_get_validity_time is None:
cls.lua_get_validity_time = client.register_script(
cls.LUA_GET_VALIDITY_TIME_SCRIPT)
def _lock_status_to_string(self) -> str:
try:
if self.__redis_lock.locked():
if self.__redis_lock.owned():
return 'locked'
return 'locked by someone else'
return 'unlocked'
except (redis_exceptions.RedisError) as exc:
return f'Error: {str(exc)}'
def testRedisNativeLock(db):
l = Lock(db.redis, 'zelock', timeout=3.0)
l.acquire()
l.release()
