def finalize(self): """ Call before you generate a dataset This caches all the guids in the db at time of calling to a list creating a data structure which allows for fast access by the dataset object and fixing the length of the the dataset """ lockname = self.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'getting lock {lockname}') self.redis.set(self.key('finalized'), 'FINALIZED') if lk.owned(): lk.release() #logger.debug(f'released lock {lockname}') self.episodes = [] self.episode_len = [] for episode in range(self.num_episodes()): self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode()) self.episodes = sorted(self.episodes) for episode_id in self.episodes: self.episode_len.append(len(Episode(self, self.redis, episode_id))) self.episode_off = [] offset = 0 for l in self.episode_len: self.episode_off.append(offset) offset += l self.finalized = True
class LockManager(): def __init__(self, lock_name, lock_timeout=10): self.lock_name = lock_name self.lock_timeout = lock_timeout self.is_lock_free = False def __enter__(self): self.lock = Lock( redis_client, self.lock_name, blocking_timeout=1, timeout=self.lock_timeout, ) try: self.is_lock_free = self.lock.acquire(blocking=False) except LockError: log.error( 'lock acquire error', extra={'data': {'lock_name': self.lock_name}}, exc_info=True, ) return self def __exit__(self, type, value, traceback): if self.is_lock_free: try: self.lock.release() except LockError: log.error( 'lock release error', extra={'data': {'lock_name': self.lock_name}}, exc_info=True, )
def end(self): if self.p is not None: self.p.execute() lockname = self.rollout.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'got lock {lockname}') if not self.redis.exists(self.rollout.key('finalized')): self.redis.lpush(self.rollout.key('episodes'), self.id) self.redis.incrby(self.rollout.key('steps'), len(self)) if lk.owned(): lk.release()
class RedisLock: ''' An awaitable redis Lock. The event loop won't be blocked when acquiring the lock. ''' def __init__(self, redi, name: str = "", sleep: int = 0.02, timeout:int = 5, blocking_timeout: int = 10, thread_local: bool = False): self._lock = RLock(redi, name, sleep=sleep, timeout=timeout, blocking_timeout=blocking_timeout, thread_local=thread_local) async def __aenter__(self): loop = asyncio.get_event_loop() await loop.run_in_executor(None, self._lock.acquire) return self async def __aexit__(self, *args): self._lock.release() async def acquire(self): '''Acquire the lock without locking the loop''' loop = asyncio.get_event_loop() return await loop.run_in_executor(None, self._lock.acquire) def release(self): '''Release the lock, this is not async for the sake of easier cleanup (e.g. registering `atexit`)''' self._lock.release()
class RedisBackendLock(DbBackendLockAbc): """ A class providing an implementation of database backend lock of Shared Data Layer (SDL), when backend database solution is Redis. Args: ns (str): Namespace under which this lock is targeted. name (str): Lock name, identifies the lock key in a Redis database backend. expiration (int, float): Lock expiration time after which the lock is removed if it hasn't been released earlier by a 'release' method. redis_backend (RedisBackend): Database backend object containing connection to Redis database. """ lua_get_validity_time = None # KEYS[1] - lock name # ARGS[1] - token # return < 0 in case of failure, otherwise return lock validity time in milliseconds. LUA_GET_VALIDITY_TIME_SCRIPT = """ local token = redis.call('get', KEYS[1]) if not token then return -10 end if token ~= ARGV[1] then return -11 end return redis.call('pttl', KEYS[1]) """ def __init__(self, ns: str, name: str, expiration: Union[int, float], redis_backend: RedisBackend) -> None: super().__init__(ns, name) self.__redis = redis_backend.get_redis_connection(ns) with _map_to_sdl_exception(): redis_lockname = '{' + ns + '},' + self._lock_name self.__redis_lock = Lock(redis=self.__redis, name=redis_lockname, timeout=expiration) self._register_scripts() def __str__(self): return str({ "lock DB type": "Redis", "lock namespace": self._ns, "lock name": self._lock_name, "lock status": self._lock_status_to_string() }) def acquire(self, retry_interval: Union[int, float] = 0.1, retry_timeout: Union[int, float] = 10) -> bool: succeeded = False self.__redis_lock.sleep = retry_interval with _map_to_sdl_exception(): succeeded = self.__redis_lock.acquire( blocking_timeout=retry_timeout) return succeeded def release(self) -> None: with _map_to_sdl_exception(): self.__redis_lock.release() def refresh(self) -> None: with _map_to_sdl_exception(): self.__redis_lock.reacquire() def get_validity_time(self) -> Union[int, float]: validity = 0 if self.__redis_lock.local.token is None: msg = u'Cannot get validity time of an unlocked lock %s' % self._lock_name raise RejectedByBackend(msg) with _map_to_sdl_exception(): validity = self.lua_get_validity_time( keys=[self.__redis_lock.name], args=[self.__redis_lock.local.token], client=self.__redis) if validity < 0: msg = ( u'Getting validity time of a lock %s failed with error code: %d' % (self._lock_name, validity)) raise RejectedByBackend(msg) ftime = validity / 1000.0 if ftime.is_integer(): return int(ftime) return ftime def _register_scripts(self): cls = self.__class__ client = self.__redis if cls.lua_get_validity_time is None: cls.lua_get_validity_time = client.register_script( cls.LUA_GET_VALIDITY_TIME_SCRIPT) def _lock_status_to_string(self) -> str: try: if self.__redis_lock.locked(): if self.__redis_lock.owned(): return 'locked' return 'locked by someone else' return 'unlocked' except (redis_exceptions.RedisError) as exc: return f'Error: {str(exc)}'
def testRedisNativeLock(db): l = Lock(db.redis, 'zelock', timeout=3.0) l.acquire() l.release()