def finalize(self): """ Call before you generate a dataset This caches all the guids in the db at time of calling to a list creating a data structure which allows for fast access by the dataset object and fixing the length of the the dataset """ lockname = self.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'getting lock {lockname}') self.redis.set(self.key('finalized'), 'FINALIZED') if lk.owned(): lk.release() #logger.debug(f'released lock {lockname}') self.episodes = [] self.episode_len = [] for episode in range(self.num_episodes()): self.episodes.append(self.redis.lindex(self.key('episodes'), episode).decode()) self.episodes = sorted(self.episodes) for episode_id in self.episodes: self.episode_len.append(len(Episode(self, self.redis, episode_id))) self.episode_off = [] offset = 0 for l in self.episode_len: self.episode_off.append(offset) offset += l self.finalized = True
def end(self): if self.p is not None: self.p.execute() lockname = self.rollout.key('lock') lk = Lock(self.redis, lockname) #logger.debug(f'getting lock {lockname}') lk.acquire() #logger.debug(f'got lock {lockname}') if not self.redis.exists(self.rollout.key('finalized')): self.redis.lpush(self.rollout.key('episodes'), self.id) self.redis.incrby(self.rollout.key('steps'), len(self)) if lk.owned(): lk.release()
class RedisBackendLock(DbBackendLockAbc): """ A class providing an implementation of database backend lock of Shared Data Layer (SDL), when backend database solution is Redis. Args: ns (str): Namespace under which this lock is targeted. name (str): Lock name, identifies the lock key in a Redis database backend. expiration (int, float): Lock expiration time after which the lock is removed if it hasn't been released earlier by a 'release' method. redis_backend (RedisBackend): Database backend object containing connection to Redis database. """ lua_get_validity_time = None # KEYS[1] - lock name # ARGS[1] - token # return < 0 in case of failure, otherwise return lock validity time in milliseconds. LUA_GET_VALIDITY_TIME_SCRIPT = """ local token = redis.call('get', KEYS[1]) if not token then return -10 end if token ~= ARGV[1] then return -11 end return redis.call('pttl', KEYS[1]) """ def __init__(self, ns: str, name: str, expiration: Union[int, float], redis_backend: RedisBackend) -> None: super().__init__(ns, name) self.__redis = redis_backend.get_redis_connection(ns) with _map_to_sdl_exception(): redis_lockname = '{' + ns + '},' + self._lock_name self.__redis_lock = Lock(redis=self.__redis, name=redis_lockname, timeout=expiration) self._register_scripts() def __str__(self): return str({ "lock DB type": "Redis", "lock namespace": self._ns, "lock name": self._lock_name, "lock status": self._lock_status_to_string() }) def acquire(self, retry_interval: Union[int, float] = 0.1, retry_timeout: Union[int, float] = 10) -> bool: succeeded = False self.__redis_lock.sleep = retry_interval with _map_to_sdl_exception(): succeeded = self.__redis_lock.acquire( blocking_timeout=retry_timeout) return succeeded def release(self) -> None: with _map_to_sdl_exception(): self.__redis_lock.release() def refresh(self) -> None: with _map_to_sdl_exception(): self.__redis_lock.reacquire() def get_validity_time(self) -> Union[int, float]: validity = 0 if self.__redis_lock.local.token is None: msg = u'Cannot get validity time of an unlocked lock %s' % self._lock_name raise RejectedByBackend(msg) with _map_to_sdl_exception(): validity = self.lua_get_validity_time( keys=[self.__redis_lock.name], args=[self.__redis_lock.local.token], client=self.__redis) if validity < 0: msg = ( u'Getting validity time of a lock %s failed with error code: %d' % (self._lock_name, validity)) raise RejectedByBackend(msg) ftime = validity / 1000.0 if ftime.is_integer(): return int(ftime) return ftime def _register_scripts(self): cls = self.__class__ client = self.__redis if cls.lua_get_validity_time is None: cls.lua_get_validity_time = client.register_script( cls.LUA_GET_VALIDITY_TIME_SCRIPT) def _lock_status_to_string(self) -> str: try: if self.__redis_lock.locked(): if self.__redis_lock.owned(): return 'locked' return 'locked by someone else' return 'unlocked' except (redis_exceptions.RedisError) as exc: return f'Error: {str(exc)}'