def finalize(self): """ Call before you generate a dataset This caches all the guids in the db at time of calling to a list creating a data structure which allows for fast access by the dataset object and fixing the length of the the dataset """ lockname = self.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'getting lock {lockname}') self.redis.set(self.key('finalized'), 'FINALIZED') if lk.owned(): lk.release() #logger.debug(f'released lock {lockname}') self.episodes = [] self.episode_len = [] for episode in range(self.num_episodes()): self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode()) self.episodes = sorted(self.episodes) for episode_id in self.episodes: self.episode_len.append(len(Episode(self, self.redis, episode_id))) self.episode_off = [] offset = 0 for l in self.episode_len: self.episode_off.append(offset) offset += l self.finalized = True
def test_redis_raise_or_lock_locked_and_expired(redis, backend): lock = RedisLock(redis, "test", timeout=1) lock.acquire() time.sleep(1) # wait for lock to expire backend.raise_or_lock(key="test", timeout=60) assert redis.get("test") is not None
def test_redis_raise_or_lock_locked(redis, backend): # Set to expire in 30 seconds! lock = RedisLock(redis, "test", timeout=30) lock.acquire() with pytest.raises(AlreadyQueued) as e: backend.raise_or_lock(key="test", timeout=60) assert e.value.countdown == 30.0 assert e.value.message == "Expires in 30.0 seconds"
def test_apply_async_expired(redis): lock = RedisLock(redis, "qo_example_a-1", timeout=1) lock.acquire() assert redis.get("qo_example_a-1") is not None time.sleep(1) example.apply_async(args=(1, )) assert redis.get("qo_example_a-1") is None
def test_delay_expired(redis): lock = RedisLock(redis, "qo_example_a-1", timeout=1) lock.acquire() assert redis.get("qo_example_a-1") is not None time.sleep(1) example.delay(1) assert redis.get("qo_example_a-1") is None
def end(self): if self.p is not None: self.p.execute() lockname = self.rollout.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'got lock {lockname}') if not self.redis.exists(self.rollout.key('finalized')): self.redis.lpush(self.rollout.key('episodes'), self.id) self.redis.incrby(self.rollout.key('steps'), len(self)) if lk.owned(): lk.release()
class LockManager(): def __init__(self, lock_name, lock_timeout=10): self.lock_name = lock_name self.lock_timeout = lock_timeout self.is_lock_free = False def __enter__(self): self.lock = Lock( redis_client, self.lock_name, blocking_timeout=1, timeout=self.lock_timeout, ) try: self.is_lock_free = self.lock.acquire(blocking=False) except LockError: log.error( 'lock acquire error', extra={'data': {'lock_name': self.lock_name}}, exc_info=True, ) return self def __exit__(self, type, value, traceback): if self.is_lock_free: try: self.lock.release() except LockError: log.error( 'lock release error', extra={'data': {'lock_name': self.lock_name}}, exc_info=True, )
def acquire(self, lox_name, lock_id, expires_seconds=None): key = self.key(lox_name, lock_id) lock = Lock(self.connection, key, timeout=expires_seconds) # retry logic is handled in core.lock, so no blocking here if lock.acquire(blocking=False): return BackendLock(key, lox_name, lock_id, provider_lock=lock) else: raise LockInUseException("Lock {} has been acquired previously, possibly by another thread/process, and is not available.".format(key))
def acquire(self, lox_name, lock_id, expires_seconds=None): key = self.key(lox_name, lock_id) lock = Lock(self.connection, key, timeout=expires_seconds) # retry logic is handled in core.lock, so no blocking here if lock.acquire(blocking=False): return BackendLock(key, lox_name, lock_id, provider_lock=lock) else: raise LockInUseException( "Lock {} has been acquired previously, possibly by another thread/process, and is not available." .format(key))
class RedisBackendLock(DbBackendLockAbc): """ A class providing an implementation of database backend lock of Shared Data Layer (SDL), when backend database solution is Redis. Args: ns (str): Namespace under which this lock is targeted. name (str): Lock name, identifies the lock key in a Redis database backend. expiration (int, float): Lock expiration time after which the lock is removed if it hasn't been released earlier by a 'release' method. redis_backend (RedisBackend): Database backend object containing connection to Redis database. """ lua_get_validity_time = None # KEYS[1] - lock name # ARGS[1] - token # return < 0 in case of failure, otherwise return lock validity time in milliseconds. LUA_GET_VALIDITY_TIME_SCRIPT = """ local token = redis.call('get', KEYS[1]) if not token then return -10 end if token ~= ARGV[1] then return -11 end return redis.call('pttl', KEYS[1]) """ def __init__(self, ns: str, name: str, expiration: Union[int, float], redis_backend: RedisBackend) -> None: super().__init__(ns, name) self.__redis = redis_backend.get_redis_connection(ns) with _map_to_sdl_exception(): redis_lockname = '{' + ns + '},' + self._lock_name self.__redis_lock = Lock(redis=self.__redis, name=redis_lockname, timeout=expiration) self._register_scripts() def __str__(self): return str({ "lock DB type": "Redis", "lock namespace": self._ns, "lock name": self._lock_name, "lock status": self._lock_status_to_string() }) def acquire(self, retry_interval: Union[int, float] = 0.1, retry_timeout: Union[int, float] = 10) -> bool: succeeded = False self.__redis_lock.sleep = retry_interval with _map_to_sdl_exception(): succeeded = self.__redis_lock.acquire( blocking_timeout=retry_timeout) return succeeded def release(self) -> None: with _map_to_sdl_exception(): self.__redis_lock.release() def refresh(self) -> None: with _map_to_sdl_exception(): self.__redis_lock.reacquire() def get_validity_time(self) -> Union[int, float]: validity = 0 if self.__redis_lock.local.token is None: msg = u'Cannot get validity time of an unlocked lock %s' % self._lock_name raise RejectedByBackend(msg) with _map_to_sdl_exception(): validity = self.lua_get_validity_time( keys=[self.__redis_lock.name], args=[self.__redis_lock.local.token], client=self.__redis) if validity < 0: msg = ( u'Getting validity time of a lock %s failed with error code: %d' % (self._lock_name, validity)) raise RejectedByBackend(msg) ftime = validity / 1000.0 if ftime.is_integer(): return int(ftime) return ftime def _register_scripts(self): cls = self.__class__ client = self.__redis if cls.lua_get_validity_time is None: cls.lua_get_validity_time = client.register_script( cls.LUA_GET_VALIDITY_TIME_SCRIPT) def _lock_status_to_string(self) -> str: try: if self.__redis_lock.locked(): if self.__redis_lock.owned(): return 'locked' return 'locked by someone else' return 'unlocked' except (redis_exceptions.RedisError) as exc: return f'Error: {str(exc)}'
def testRedisNativeLock(db): l = Lock(db.redis, 'zelock', timeout=3.0) l.acquire() l.release()