def greenlet_queuestats(self):
interval = min(self.config["orchestrate_interval"], 1 * 60)
lock_timeout = 5 * 60 + (interval * 2)
while True:
lock = LuaLock(connections.redis,
self.redis_queuestats_lock_key,
timeout=lock_timeout,
thread_local=False,
blocking=False)
with lock:
lock_expires = time.time() + lock_timeout
self.queue_etas = defaultdict(lambda: MovingETA(5))
while True:
self.queuestats()
# Because queue stats can be expensive, we try to keep the lock on the same agent
lock_extend = (time.time() + lock_timeout) - lock_expires
lock_expires += lock_extend
lock.extend(lock_extend)
time.sleep(interval)
time.sleep(interval)
def lock(self, name, timeout=None, sleep=0.1, blocking_timeout=None,
lock_class=None, thread_local=True):
"""
Return a new Lock object using key ``name`` that mimics
the behavior of threading.Lock.
If specified, ``timeout`` indicates a maximum life for the lock.
By default, it will remain locked until release() is called.
``sleep`` indicates the amount of time to sleep per loop iteration
when the lock is in blocking mode and another client is currently
holding the lock.
``blocking_timeout`` indicates the maximum amount of time in seconds to
spend trying to acquire the lock. A value of ``None`` indicates
continue trying forever. ``blocking_timeout`` can be specified as a
float or integer, both representing the number of seconds to wait.
``lock_class`` forces the specified lock implementation.
``thread_local`` indicates whether the lock token is placed in
thread-local storage. By default, the token is placed in thread local
storage so that a thread only sees its token, not a token set by
another thread. Consider the following timeline:
time: 0, thread-1 acquires `my-lock`, with a timeout of 5 seconds.
thread-1 sets the token to "abc"
time: 1, thread-2 blocks trying to acquire `my-lock` using the
Lock instance.
time: 5, thread-1 has not yet completed. redis expires the lock
key.
time: 5, thread-2 acquired `my-lock` now that it's available.
thread-2 sets the token to "xyz"
time: 6, thread-1 finishes its work and calls release(). if the
token is *not* stored in thread local storage, then
thread-1 would see the token value as "xyz" and would be
able to successfully release the thread-2's lock.
In some use cases it's necessary to disable thread local storage. For
example, if you have code where one thread acquires a lock and passes
that lock instance to a worker thread to release later. If thread
local storage isn't disabled in this case, the worker thread won't see
the token set by the thread that acquired the lock. Our assumption
is that these cases aren't common and as such default to using
thread local storage. """
if lock_class is None:
if self._use_lua_lock is None:
# the first time .lock() is called, determine if we can use
# Lua by attempting to register the necessary scripts
try:
LuaLock.register_scripts(self)
self._use_lua_lock = True
except ResponseError:
self._use_lua_lock = False
lock_class = self._use_lua_lock and LuaLock or Lock
return lock_class(self, name, timeout=timeout, sleep=sleep,
blocking_timeout=blocking_timeout,
thread_local=thread_local)
def greenlet_scheduler(self):
redis_scheduler_lock_key = "%s:schedulerlock" % get_current_config()["redis_prefix"]
while True:
with LuaLock(connections.redis, redis_scheduler_lock_key,
timeout=self.config["scheduler_interval"] + 10, blocking=False, thread_local=False):
self.scheduler.check()
time.sleep(self.config["scheduler_interval"])
def release_lock(lock_name, token):
"""
Release a lock
Args:
lock_name (str): The lock key in redis
token (bytes): The unique id used
Returns:
bool: True if the lock was successfully released
"""
# this is a StrictRedis instance, we need this for the script installation that LuaLock uses
redis = caches['redis'].client.get_client()
lock = LuaLock(redis, lock_name)
try:
lock.do_release(token)
except LockError:
# If the lock is expired we don't want to raise an error
pass
def release_lock(lock_name, token):
"""
Release a lock
Args:
lock_name (str): The lock key in redis
token (bytes): The unique id used
Returns:
bool: True if the lock was successfully released
"""
# this is a StrictRedis instance, we need this for the script installation that LuaLock uses
redis = caches['redis'].client.get_client()
lock = LuaLock(redis, lock_name)
try:
lock.do_release(token)
except LockError:
# If the lock is expired we don't want to raise an error
pass
def _get_lock(lock_name, expiration):
"""
Creates a new redis LuaLock
Args:
lock_name (str): The name of the lock
expiration (datetime.datetime): The expiration datetime
Returns:
redis.lock.Lock: a redis lua-based lock
"""
timeout = int((expiration - now_in_utc()).total_seconds())
# this is a StrictRedis instance, we need this for the script installation that LuaLock uses
redis = caches['redis'].client.get_client()
# don't block acquiring the lock, the task will need to try again later
return LuaLock(redis,
lock_name,
timeout=timeout,
blocking=False,
thread_local=False)