def get_concurrent_redshift_query_queue_semaphore(queue_name):
concurrency = settings.REDSHIFT_QUERY_QUEUES[queue_name]["concurrency"]
concurrent_redshift_query_semaphore = Semaphore(
redshift.get_redshift_cache_redis_client(),
count=concurrency,
namespace=queue_name,
stale_client_timeout=300,
blocking=False)
return concurrent_redshift_query_semaphore
def _do_execute_query(parameterized_query, wlm_queue=None):
# This method should always be getting executed within a Lambda context
# Distributed dog pile lock pattern
# From: https://pypi.python.org/pypi/python-redis-lock
log.info("About to attempt acquiring lock...")
redis_client = redshift.get_redshift_cache_redis_client()
with RedisLock(redis_client, parameterized_query.cache_key, expire=300):
# Get a lock with a 5-minute lifetime since that's the maximum duration of a Lambda
# to ensure the lock is held for as long as the Python process / Lambda is running.
log.info("Lock acquired.")
return _do_execute_query_work(parameterized_query, wlm_queue)
def _lock_exists(cache_key):
lock_signal_key = _get_lock_signal_key(cache_key)
redis_client = redshift.get_redshift_cache_redis_client()
lock_signal = redis_client.get(lock_signal_key)
return lock_signal is not None
def evict_locks_cache(params):
redis_client = redshift.get_redshift_cache_redis_client()
lock_signal_key = _get_lock_signal_key(params.cache_key)
redis_client.delete(lock_signal_key)