def greenlet_queuestats(self): interval = min(self.config["orchestrate_interval"], 1 * 60) lock_timeout = 5 * 60 + (interval * 2) while True: lock = LuaLock(connections.redis, self.redis_queuestats_lock_key, timeout=lock_timeout, thread_local=False, blocking=False) with lock: lock_expires = time.time() + lock_timeout self.queue_etas = defaultdict(lambda: MovingETA(5)) while True: self.queuestats() # Because queue stats can be expensive, we try to keep the lock on the same agent lock_extend = (time.time() + lock_timeout) - lock_expires lock_expires += lock_extend lock.extend(lock_extend) time.sleep(interval) time.sleep(interval)
def lock(self, name, timeout=None, sleep=0.1, blocking_timeout=None, lock_class=None, thread_local=True): """ Return a new Lock object using key ``name`` that mimics the behavior of threading.Lock. If specified, ``timeout`` indicates a maximum life for the lock. By default, it will remain locked until release() is called. ``sleep`` indicates the amount of time to sleep per loop iteration when the lock is in blocking mode and another client is currently holding the lock. ``blocking_timeout`` indicates the maximum amount of time in seconds to spend trying to acquire the lock. A value of ``None`` indicates continue trying forever. ``blocking_timeout`` can be specified as a float or integer, both representing the number of seconds to wait. ``lock_class`` forces the specified lock implementation. ``thread_local`` indicates whether the lock token is placed in thread-local storage. By default, the token is placed in thread local storage so that a thread only sees its token, not a token set by another thread. Consider the following timeline: time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds. thread-1 sets the token to "abc" time: 1, thread-2 blocks trying to acquire `my-lock` using the Lock instance. time: 5, thread-1 has not yet completed. redis expires the lock key. time: 5, thread-2 acquired `my-lock` now that it's available. thread-2 sets the token to "xyz" time: 6, thread-1 finishes its work and calls release(). if the token is *not* stored in thread local storage, then thread-1 would see the token value as "xyz" and would be able to successfully release the thread-2's lock. In some use cases it's necessary to disable thread local storage. For example, if you have code where one thread acquires a lock and passes that lock instance to a worker thread to release later. If thread local storage isn't disabled in this case, the worker thread won't see the token set by the thread that acquired the lock. Our assumption is that these cases aren't common and as such default to using thread local storage. """ if lock_class is None: if self._use_lua_lock is None: # the first time .lock() is called, determine if we can use # Lua by attempting to register the necessary scripts try: LuaLock.register_scripts(self) self._use_lua_lock = True except ResponseError: self._use_lua_lock = False lock_class = self._use_lua_lock and LuaLock or Lock return lock_class(self, name, timeout=timeout, sleep=sleep, blocking_timeout=blocking_timeout, thread_local=thread_local)
def greenlet_scheduler(self): redis_scheduler_lock_key = "%s:schedulerlock" % get_current_config()["redis_prefix"] while True: with LuaLock(connections.redis, redis_scheduler_lock_key, timeout=self.config["scheduler_interval"] + 10, blocking=False, thread_local=False): self.scheduler.check() time.sleep(self.config["scheduler_interval"])
def release_lock(lock_name, token): """ Release a lock Args: lock_name (str): The lock key in redis token (bytes): The unique id used Returns: bool: True if the lock was successfully released """ # this is a StrictRedis instance, we need this for the script installation that LuaLock uses redis = caches['redis'].client.get_client() lock = LuaLock(redis, lock_name) try: lock.do_release(token) except LockError: # If the lock is expired we don't want to raise an error pass
def _get_lock(lock_name, expiration): """ Creates a new redis LuaLock Args: lock_name (str): The name of the lock expiration (datetime.datetime): The expiration datetime Returns: redis.lock.Lock: a redis lua-based lock """ timeout = int((expiration - now_in_utc()).total_seconds()) # this is a StrictRedis instance, we need this for the script installation that LuaLock uses redis = caches['redis'].client.get_client() # don't block acquiring the lock, the task will need to try again later return LuaLock(redis, lock_name, timeout=timeout, blocking=False, thread_local=False)