def test_token(redis_server):
conn = StrictRedis(unix_socket_path=UDS_PATH)
lock = Lock(conn, "foobar-tok")
tok = lock.id
assert conn.get(lock._name) is None
lock.acquire(blocking=False)
assert conn.get(lock._name) == tok
def continuous_migration(skip_files=None):
"""Task to continuously migrate what is pushed up by Legacy."""
if skip_files is None:
skip_files = current_app.config.get(
'RECORDS_MIGRATION_SKIP_FILES',
False,
)
redis_url = current_app.config.get('CACHE_REDIS_URL')
r = StrictRedis.from_url(redis_url)
lock = Lock(r, 'continuous_migration', expire=120, auto_renewal=True)
if lock.acquire(blocking=False):
try:
while r.llen('legacy_records'):
raw_record = r.lrange('legacy_records', 0, 0)
if raw_record:
migrate_and_insert_record(
zlib.decompress(raw_record[0]),
skip_files=skip_files,
)
db.session.commit()
r.lpop('legacy_records')
finally:
lock.release()
else:
LOGGER.info("Continuous_migration already executed. Skipping.")
def test_signal_cleanup_on_reset(conn):
"""After resetting a lock, the signal key should not remain."""
lock = Lock(conn, 'foo')
lock.acquire()
lock.reset()
assert conn.llen('lock-signal:foo') == 0
assert conn.exists('lock-signal:foo') == 0
def test_signal_expiration(conn):
"""Signal keys expire within two seconds after releasing the lock."""
lock = Lock(conn, 'signal_expiration')
lock.acquire()
lock.release()
time.sleep(2)
assert conn.llen('lock-signal:signal_expiration') == 0
def test_signal_cleanup_on_reset_all(conn):
"""After resetting all locks, no signal keys should not remain."""
lock = Lock(conn, 'foo')
lock.acquire()
reset_all(conn)
assert conn.llen('lock-signal:foo') == 0
assert conn.exists('lock-signal:foo') == 0
def _execute_task_group(self, queue, tasks, all_task_ids):
"""
Executes the given tasks in the queue. Updates the heartbeat for task
IDs passed in all_task_ids. This internal method is only meant to be
called from within _process_from_queue.
"""
log = self.log.bind(queue=queue)
locks = []
# Keep track of the acquired locks: If two tasks in the list require
# the same lock we only acquire it once.
lock_ids = set()
ready_tasks = []
for task in tasks:
if task.get('lock', False):
if task.get('lock_key'):
kwargs = task.get('kwargs', {})
lock_id = gen_unique_id(
task['func'],
None,
{key: kwargs.get(key) for key in task['lock_key']},
)
else:
lock_id = gen_unique_id(
task['func'],
task.get('args', []),
task.get('kwargs', {}),
)
if lock_id not in lock_ids:
lock = Lock(self.connection, self._key('lock', lock_id), timeout=self.config['ACTIVE_TASK_UPDATE_TIMEOUT'])
acquired = lock.acquire(blocking=False)
if acquired:
lock_ids.add(lock_id)
locks.append(lock)
else:
log.info('could not acquire lock', task_id=task['id'])
# Reschedule the task
when = time.time() + self.config['LOCK_RETRY']
self._redis_move_task(queue, task['id'], ACTIVE, SCHEDULED, when)
# Make sure to remove it from this list so we don't re-add
# to the ACTIVE queue by updating the heartbeat.
all_task_ids.remove(task['id'])
continue
ready_tasks.append(task)
if not ready_tasks:
return True, []
success = self._execute(queue, ready_tasks, log, locks, all_task_ids)
for lock in locks:
lock.release()
return success, ready_tasks
def lock_redis(app):
redis_url = app.config.get('CACHE_REDIS_URL')
redis = StrictRedis.from_url(redis_url)
lock = Lock(redis, 'my_lock', expire=60)
lock.acquire(blocking=False)
yield
lock.release()
def workerfn(go, count_lock, count):
redis_lock = Lock(make_conn(), 'lock')
with count_lock:
count.value += 1
go.wait()
if redis_lock.acquire(blocking=True):
with count_lock:
count.value += 1
def test_timeout_acquired(conn):
with TestProcess(sys.executable, HELPER, 'test_timeout') as proc:
with dump_on_error(proc.read):
name = 'lock:foobar'
wait_for_strings(
proc.read, TIMEOUT,
'Getting %r ...' % name,
'Got lock for %r.' % name,
)
lock = Lock(conn, "foobar")
assert lock.acquire(timeout=2)
def wrapper(*args, **kwargs):
client = get_redis_connection()
lock = Lock(client, lock_name, expire=expire)
try:
if lock.acquire(blocking=False):
return func(*args, **kwargs)
else:
logger.warning('another instance of %s is running',
func_path)
finally:
lock.release()
def test_timeout_acquired(conn):
with TestProcess(sys.executable, HELPER, 'test_timeout') as proc:
with dump_on_error(proc.read):
name = 'lock:foobar'
wait_for_strings(
proc.read,
TIMEOUT,
'Getting %r ...' % name,
'Got lock for %r.' % name,
)
lock = Lock(conn, "foobar")
assert lock.acquire(timeout=2)
def test_auto_renewal(conn):
lock = Lock(conn, 'lock_renewal', expire=3, auto_renewal=True)
lock.acquire()
assert isinstance(lock._lock_renewal_thread, InterruptableThread)
assert not lock._lock_renewal_thread.should_exit
assert lock._lock_renewal_interval == 2
time.sleep(3)
assert conn.get(lock._name) == lock.id, "Key expired but it should have been getting renewed"
lock.release()
assert lock._lock_renewal_thread is None
def test_expire_int_conversion():
conn = object()
lock = Lock(conn, name='foobar', strict=False, expire=1)
assert lock._expire == 1
lock = Lock(conn, name='foobar', strict=False, expire=0)
assert lock._expire is None
lock = Lock(conn, name='foobar', strict=False, expire="1")
assert lock._expire == 1
lock = Lock(conn, name='foobar', strict=False, expire="123")
assert lock._expire == 123
def test_expire_less_than_timeout(conn):
lock = Lock(conn, "foobar", expire=1)
pytest.raises(TimeoutTooLarge, lock.acquire, blocking=True, timeout=2)
lock = Lock(conn, "foobar", expire=1, auto_renewal=True)
lock.acquire(blocking=True, timeout=2)
lock.release()
def test_expire_without_timeout(conn):
first_lock = Lock(conn, 'expire', expire=2)
second_lock = Lock(conn, 'expire', expire=1)
first_lock.acquire()
assert second_lock.acquire(blocking=False) is False
assert second_lock.acquire() is True
second_lock.release()
def test_auto_renewal(conn):
lock = Lock(conn, 'lock_renewal', expire=3, auto_renewal=True)
lock.acquire()
assert isinstance(lock._lock_renewal_thread, threading.Thread)
assert not lock._lock_renewal_stop.is_set()
assert isinstance(lock._lock_renewal_interval, float)
assert lock._lock_renewal_interval == 2
time.sleep(3)
assert maybe_decode(conn.get(lock._name)) == lock.id, "Key expired but it should have been getting renewed"
lock.release()
assert lock._lock_renewal_thread is None
def test_reset(conn):
lock = Lock(conn, "foobar")
lock.reset()
new_lock = Lock(conn, "foobar")
new_lock.acquire(blocking=False)
new_lock.release()
pytest.raises(NotAcquired, lock.release)
def test_invalid_timeout(conn):
lock = Lock(conn, "foobar")
with pytest.raises(InvalidTimeout):
lock.acquire(blocking=True, timeout=0)
lock = Lock(conn, "foobar")
with pytest.raises(InvalidTimeout):
lock.acquire(blocking=True, timeout=-1)
def get():
if not cache.exists('branches'):
with Lock(cache, 'lock-branches'):
if not cache.exists('branches'):
replace()
return json.loads(cache.get('branches'))
def say_w_beep(speaker, speech_text):
redis_host = os.getenv('REDIS_HOST', "127.0.0.1")
conn = StrictRedis(host=redis_host)
# Synthesize the sample text, saving it in an MP3 audio file
polly_client = boto3.client('polly')
response = polly_client.synthesize_speech(VoiceId=speaker,
Engine='neural',
OutputFormat='mp3',
Text=speech_text)
with Lock(conn, "talking"):
print("Got the lock. Doing some work ...")
pygame.mixer.init()
pygame.mixer.music.load("beep.mp3")
pygame.mixer.music.play()
# Wait for Auudio to Finish
while pygame.mixer.music.get_busy() == True:
continue
pygame.mixer.music.load(response['AudioStream'])
pygame.mixer.music.play()
# Wait for Auudio to Finish
while pygame.mixer.music.get_busy() == True:
continue
def get_secret_key():
# To avoid each installation having the same Django SECERT_KEY we generate
# a random one and store it in Redis. We have to store it somewhere
# central like Redis because if each worker generated it's own it would
# cause problems (like JWT "Error decoding signature").
secret_key = None
if 'DJANGO_SECRET_KEY' in os.environ:
secret_key = os.environ.get('DJANGO_SECRET_KEY')
else:
r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
if r.exists('django_secret_key'):
secret_key = r.get('django_secret_key').decode('utf-8')
else:
# Make sure only first worker generates the key and others get from Redis
with Lock(r, 'django_secret_key_generation_lock'):
if r.exists('django_secret_key'):
secret_key = r.get('django_secret_key').decode('utf-8')
else:
secret_key = utils.get_random_secret_key()
r.set('django_secret_key', secret_key.encode('utf-8'))
if not secret_key:
raise EnvironmentError('No secret key available')
return secret_key
def load_graph(self, graph_file):
with Lock(redis_connection,
'classifier_{}_load_graph'.format(self.name)):
# Load MTCNN
mtcnn_graph = None
mtcnn_key = '{self.graph_cache_key}:mtcnn'
if mtcnn_key in self.graph_cache:
mtcnn_graph = self.graph_cache[mtcnn_key]
else:
mtcnn_graph = MTCNN(weights_file=graph_file)
self.graph_cache[mtcnn_key] = mtcnn_graph
# Load Facenet
facenet_graph = None
facenet_key = '{self.graph_cache_key}:facenet'
if facenet_key in self.graph_cache:
facenet_graph = self.graph_cache[facenet_key]
else:
facenet_graph = build_model('Facenet')
self.graph_cache[facenet_key] = facenet_graph
# Store version number of retrained model (ANN) if it has been computed
self.reload_retrained_model_version()
return {
'mtcnn': mtcnn_graph,
'facenet': facenet_graph,
}
def test_given_id(conn):
"""It is possible to extend a lock using another instance of Lock with the
same name.
"""
name = 'foobar'
key_name = 'lock:' + name
orig = Lock(conn, name, expire=100, id=b"a")
orig.acquire()
pytest.raises(TypeError, Lock, conn, name, id=object())
lock = Lock(conn, name, id=b"a")
pytest.raises(AlreadyAcquired, lock.acquire)
lock.extend(100)
lock.release() # this works, note that this ain't the object that acquired the lock
pytest.raises(NotAcquired, orig.release) # and this fails because lock was released above
assert conn.ttl(key_name) == -2
def update(self, comments: ["Comment"]):
"""
Update sorted comments in cache
This should be called on votes (maybe not all of them) and on new comments
:param link_id: link id
:param comment: comment
"""
for comment in comments:
cache_key = self._cache_key(comment.parent_id)
lock_key = self._lock_key(comment.parent_id)
# update comment under read - write - modify lock
with Lock(cache.conn, lock_key):
# maybe check against the comment tree to see if it is missing or it just is not initialized yet
comments = (
cache.get(cache_key) or []
) # so maybe load comments instead of []
# update comment
for i in range(len(comments)):
if comments[i][0] == comment.id:
comments[i] = [
comment.id,
confidence(comment.ups, comment.downs),
]
break
else:
# add comment
comments.append(
[comment.id, confidence(comment.ups, comment.downs)]
)
# sort and save
comments = sorted(comments, key=lambda x: x[1:], reverse=True)
cache.set(cache_key, comments)
def find_closest_face_tag_by_ann(self, source_embedding):
# Use ANN index to do quick serach if it has been trained by retrain_face_similarity_index
from django.conf import settings
ann_path = Path(
settings.MODEL_DIR) / 'face' / f'{self.library_id}_faces.ann'
tag_ids_path = Path(
settings.MODEL_DIR
) / 'face' / f'{self.library_id}_faces_tag_ids.json'
if os.path.exists(ann_path) and os.path.exists(tag_ids_path):
embedding_size = 128 # FaceNet output size
t = AnnoyIndex(embedding_size, 'euclidean')
# Ensure ANN index, tag IDs and version files can't be updated while we are reading
r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
with Lock(r, 'face_model_retrain'):
self.reload_retrained_model_version()
t.load(str(ann_path))
with open(tag_ids_path) as f:
tag_ids = json.loads(f.read())
nearest = t.get_nns_by_vector(source_embedding,
1,
include_distances=True)
if nearest[0]:
return tag_ids[nearest[0][0]], nearest[1][0]
return (None, 999)
def load_graph(self, graph_file):
r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
with Lock(r, 'classifier_{}_load_graph'.format(self.name)):
# Load MTCNN
mtcnn_graph = None
mtcnn_key = '{self.graph_cache_key}:mtcnn'
if mtcnn_key in self.graph_cache:
mtcnn_graph = self.graph_cache[mtcnn_key]
else:
mtcnn_graph = MTCNN(weights_file=graph_file)
self.graph_cache[mtcnn_key] = mtcnn_graph
# Load Facenet
facenet_graph = None
facenet_key = '{self.graph_cache_key}:facenet'
if facenet_key in self.graph_cache:
facenet_graph = self.graph_cache[facenet_key]
else:
facenet_graph = build_model('Facenet')
self.graph_cache[facenet_key] = facenet_graph
# Store version number of retrained model (ANN) if it has been computed
self.reload_retrained_model_version()
return {
'mtcnn': mtcnn_graph,
'facenet': facenet_graph,
}
def __init__(
self,
lock_path: str,
scope: str,
key: int,
slug: str,
cache_template: str = "lock:{scope}:{key}",
file_template: Optional[str] = "{slug}-{scope}.lock",
timeout: int = 1,
):
self._timeout = timeout
self._lock_path = lock_path
self._scope = scope
self._key = key
self._slug = slug
self._depth = 0
default_cache = caches["default"]
self.use_redis = isinstance(default_cache, RedisCache)
if self.use_redis:
# Prefer Redis locking as it works distributed
self._lock = Lock(
default_cache.client.get_client(),
name=self._format_template(cache_template),
expire=6,
auto_renewal=True,
)
else:
# Fall back to file based locking
self._lock = FileLock(
os.path.join(lock_path, self._format_template(file_template)),
timeout=self._timeout,
)
def reset():
redis = StrictRedis(connection_pool=__Application.redis_pool)
lock1 = Lock(redis, "Server-Empty")
lock2 = Lock(redis, "Server-Overview")
if lock1.acquire(blocking=False):
try:
redis.delete("Empty")
reset_empty()
finally:
lock1.release()
if lock2.acquire(blocking=False):
try:
redis.delete("Overview")
reset_overview()
finally:
lock2.release()
def merge_duplicate_tracks(*args, **kwargs):
track_query = Track.query.filter(*args, **kwargs).order_by(Track.id)
count = track_query.count()
tracks = track_query.all()
track_id = int(tracks[0].id)
def delete_track(track):
lock = Lock(redis_conn, 'track_{}'.format(track.id), expire=60,
auto_renewal=True)
if lock.acquire(timeout=1):
ret = db.session.delete(track)
lock.release()
return ret
else:
return False
if len(tracks) > 1:
with Lock(redis_conn, 'track_{}'.format(track_id), expire=60,
auto_renewal=True):
# update TrackLogs
TrackLog.query.filter(TrackLog.track_id.in_(
[track.id for track in tracks[1:]])).update(
{TrackLog.track_id: track_id}, synchronize_session=False)
# delete existing Track entries
map(delete_track, tracks[1:])
db.session.commit()
return count, track_id
def load_data(self) -> None:
dossiers = get_dossiers_legislatifs(*self.legislatures)
organes, acteurs = get_organes_acteurs()
with Lock(self.connection, "data"):
self.connection.set("dossiers", pickle.dumps(dossiers))
self.connection.set("organes", pickle.dumps(organes))
self.connection.set("acteurs", pickle.dumps(acteurs))
def get_read_modify_write_lock(self) -> Lock:
"""
Gets read/modify/write lock for given things
Used when updating in cache or database
:return: RedisLock
"""
return Lock(cache.conn, self._lock_key, expire=3)
def handle(self, *args, **options):
try:
while True:
with Lock(redis_connection, 'rescan_photos'):
self.rescan_photos(options['paths'])
sleep(60 * 60) # Sleep for an hour
except KeyboardInterrupt:
pass
def redis_locking_context(lock_name, expire=120, auto_renewal=True):
"""Locked Context Manager to perform operations on Redis."""
if not lock_name:
raise RedisLockError('Lock name not specified.')
redis_url = app.config.get('CACHE_REDIS_URL')
redis = StrictRedis.from_url(redis_url)
lock = Lock(redis, lock_name, expire=expire, auto_renewal=auto_renewal)
if lock.acquire(blocking=False):
try:
yield redis
finally:
lock.release()
else:
raise RedisLockError('Can not acquire Redis lock for %s', lock_name)
def continuous_migration():
"""Task to continuously migrate what is pushed up by Legacy."""
redis_url = current_app.config.get('CACHE_REDIS_URL')
r = StrictRedis.from_url(redis_url)
lock = Lock(r, 'continuous_migration', expire=120, auto_renewal=True)
if lock.acquire(blocking=False):
try:
while r.llen('legacy_records'):
raw_record = r.lrange('legacy_records', 0, 0)
if raw_record:
migrate_and_insert_record(zlib.decompress(raw_record[0]))
db.session.commit()
r.lpop('legacy_records')
finally:
lock.release()
else:
logger.info("Continuous_migration already executed. Skipping.")
def get_data(self, key: str) -> dict:
with Lock(self.connection, "data"):
raw_bytes = self.connection.get(key)
if raw_bytes is None:
return {}
data: dict = pickle.loads(raw_bytes)
return data
def test_extend_lock_without_expire_fail(conn):
name = 'foobar'
with Lock(conn, name) as lock:
with pytest.raises(NotExpirable):
lock.extend(expire=1000)
with pytest.raises(TypeError):
lock.extend()
def redis_locking_context(lock_name, expire=120, auto_renewal=True):
"""Locked Context Manager to perform operations on Redis."""
if not lock_name:
raise RedisLockError('Lock name not specified.')
redis_url = app.config.get('CACHE_REDIS_URL')
redis = StrictRedis.from_url(redis_url)
lock = Lock(redis, lock_name, expire=expire, auto_renewal=auto_renewal)
if lock.acquire(blocking=False):
try:
yield redis
finally:
lock.release()
else:
raise RedisLockError('Can not acquire Redis lock for %s', lock_name)
def test_extend(conn):
name = 'foobar'
key_name = 'lock:' + name
with Lock(conn, name, expire=100) as lock:
assert conn.ttl(key_name) <= 100
lock.extend(expire=1000)
assert conn.ttl(key_name) > 100
def test_extend_lock_default_expire(conn):
name = 'foobar'
key_name = 'lock:' + name
with Lock(conn, name, expire=1000) as lock:
time.sleep(3)
assert conn.ttl(key_name) <= 997
lock.extend()
assert 997 < conn.ttl(key_name) <= 1000
def test_reset(redis_server):
conn = StrictRedis(unix_socket_path=UDS_PATH)
with Lock(conn, "foobar") as lock:
lock.reset()
new_lock = Lock(conn, "foobar")
new_lock.acquire(blocking=False)
new_lock.release()
def test_expire(self):
conn = StrictRedis(unix_socket_path=UDS_PATH)
with Lock(conn, "foobar", expire=TIMEOUT/4):
with TestProcess(sys.executable, __file__, 'daemon', 'test_expire') as proc:
with self.dump_on_error(proc.read):
name = 'lock:foobar'
self.wait_for_strings(proc.read, TIMEOUT,
'Getting %r ...' % name,
'Got lock for %r.' % name,
'Releasing %r.' % name,
'UNLOCK_SCRIPT not cached.',
'DIED.',
)
lock = Lock(conn, "foobar")
try:
self.assertEqual(lock.acquire(blocking=False), True)
finally:
lock.release()
def test_expire(conn):
with Lock(conn, "foobar", expire=TIMEOUT/4):
with TestProcess(sys.executable, HELPER, 'test_expire') as proc:
with dump_on_error(proc.read):
name = 'lock:foobar'
wait_for_strings(
proc.read, TIMEOUT,
'Getting %r ...' % name,
'Got lock for %r.' % name,
'Releasing %r.' % name,
'UNLOCK_SCRIPT not cached.',
'DIED.',
)
lock = Lock(conn, "foobar")
try:
assert lock.acquire(blocking=False) == True
finally:
lock.release()
def handle(self, *args, **options):
r = redis.Redis(host=os.environ.get('REDIS_HOST', '127.0.0.1'))
try:
while True:
with Lock(r, 'rescan_photos'):
self.rescan_photos(options['paths'])
sleep(60 * 60) # Sleep for an hour
except KeyboardInterrupt:
pass
def legacy_orcid_arrays():
"""Generator to fetch token data from redis.
Yields:
list: user data in the form of [orcid, token, email, name]
"""
redis_url = current_app.config.get('CACHE_REDIS_URL')
r = StrictRedis.from_url(redis_url)
lock = Lock(r, 'import_legacy_orcid_tokens', expire=120, auto_renewal=True)
if lock.acquire(blocking=False):
try:
while r.llen('legacy_orcid_tokens'):
yield loads(r.lrange('legacy_orcid_tokens', 0, 1)[0])
r.lpop('legacy_orcid_tokens')
finally:
lock.release()
else:
logger.info("Import_legacy_orcid_tokens already executed. Skipping.")
def test_locked_method(conn):
lock_name = 'lock_name'
lock = Lock(conn, lock_name, id='first')
another_lock = Lock(conn, lock_name, id='another')
assert lock.locked() is False
assert another_lock.locked() is False
assert lock.acquire() is True
# another lock has same name and different id,
# but method returns true
assert lock.locked() is True
assert another_lock.locked() is True
def test_owner_id(conn):
unique_identifier = b"foobar-identifier"
lock = Lock(conn, "foobar-tok", expire=TIMEOUT/4, id=unique_identifier)
lock_id = lock.id
assert lock_id == unique_identifier
lock.acquire(blocking=False)
assert lock.get_owner_id() == unique_identifier
lock.release()
def main():
redis = StrictRedis(connection_pool=__Application.redis_pool)
lock = Lock(redis, "Spider")
try:
if lock.acquire(blocking=False):
try:
logging.info("开始课程信息收集工作...")
logging.info("初始化工作环境...")
redis.delete("Spider")
# 采集基础信息
prepare()
# 采集详细信息
core()
# 校正并归并数据
correct_and_merge()
# 将数据放入生产环境
service.copy_to_pro() if os.getenv("env") == "pro" else None
now = time.time() * 1000
logging.info(
"本轮课程信息收集工作成功完成. 共计耗时 %f seconds",
(int(now) - int(os.getenv("startup_time"))) / 1000
)
finally:
lock.release()
else:
logging.warning("Terminated for another process locked [%s]", "Spider")
except SystemExit or KeyboardInterrupt as e:
raise e
except Exception as e:
__Application.send_email(
subject="【南师教室】错误报告",
message=f"{type(e), e}\n"
f"{e.__traceback__.tb_frame.f_globals['__file__']}:{e.__traceback__.tb_lineno}\n"
)
logging.error(f"{type(e), e}")
logging.info("Exit with code %d", -1)
exit(-1)
finally:
# 从Redis清除缓存数据
redis.delete("Spider")
def test_expire(conn):
with Lock(conn, "foobar", expire=TIMEOUT / 4):
with TestProcess(sys.executable, HELPER, "test_expire") as proc:
with dump_on_error(proc.read):
name = "lock:foobar"
wait_for_strings(
proc.read,
TIMEOUT,
"Getting %r ..." % name,
"Got lock for %r." % name,
"Releasing %r." % name,
"UNLOCK_SCRIPT not cached.",
"DIED.",
)
lock = Lock(conn, "foobar")
try:
assert lock.acquire(blocking=False) == True
finally:
lock.release()
def handle_empty(args: Dict[str, str]) -> Dict[str, Any]:
if not __Application.serve:
return {
'status': 1,
'message': "service off",
'service': "off",
'data': []
}
redis = StrictRedis(connection_pool=__Application.redis_pool)
lock = Lock(redis, "Server-Empty")
if lock.acquire():
try:
if 'day' not in args.keys() or not args['day'].isdigit() or not (
0 <= int(args['day']) <= 6):
raise KeyError('day')
elif 'dqjc' not in args.keys() or not args['dqjc'].isdigit():
raise KeyError('dqjc')
elif 'jxl' not in args.keys() or not redis.hexists(
"Empty", f"{args['jxl']}_{args['day']}"):
raise KeyError('jxl')
jxl, day, dqjc = args['jxl'], int(args['day']), int(args['dqjc'])
value = json.loads(
redis.hget(name="Empty", key=f"{args['jxl']}_{args['day']}"))
classrooms = []
for classroom in value:
if classroom['jc_ks'] <= dqjc <= classroom['jc_js']:
classrooms.append(classroom)
for i in range(len(classrooms)):
classrooms[i]['id'] = i + 1
return {
'status': 0,
'message': "ok",
'service': "on",
'data': classrooms
}
finally:
lock.release()
def test_reset_signalizes(make_conn, make_process):
"""Call to reset() causes LPUSH to signal key, so blocked waiters
become unblocked."""
def workerfn(unblocked):
conn = make_conn()
lock = Lock(conn, 'lock')
if lock.acquire():
unblocked.value = 1
unblocked = multiprocessing.Value('B', 0)
conn = make_conn()
lock = Lock(conn, 'lock')
lock.acquire()
worker = make_process(target=workerfn, args=(unblocked,))
worker.start()
worker.join(0.5)
lock.reset()
worker.join(0.5)
assert unblocked.value == 1
def test_auto_renewal_stops_after_gc(conn):
"""Auto renewal stops after lock is garbage collected."""
lock = Lock(conn, 'spam', auto_renewal=True, expire=1)
name = lock._name
lock.acquire(blocking=True)
lock_renewal_thread = lock._lock_renewal_thread
del lock
gc.collect()
slept = 0
interval = 0.1
while slept <= 5:
slept += interval
lock_renewal_thread.join(interval)
if not lock_renewal_thread.is_alive():
break
time.sleep(1.5)
assert not lock_renewal_thread.is_alive()
assert conn.get(name) is None
def test_owner_id(conn):
unique_identifier = b"foobar-identifier"
lock = Lock(conn, "foobar-tok", expire=TIMEOUT/4, id=unique_identifier)
lock_id = lock.id
assert lock_id == unique_identifier
lock.acquire(blocking=False)
assert lock.get_owner_id() == unique_identifier
lock.release()
def test_invalid_timeout(conn):
lock = Lock(conn, "foobar")
with pytest.raises(InvalidTimeout):
lock.acquire(blocking=True, timeout=0)
lock = Lock(conn, "foobar")
with pytest.raises(InvalidTimeout):
lock.acquire(blocking=True, timeout=-1)
def test_given_id(conn):
"""It is possible to extend a lock using another instance of Lock with the
same name.
"""
name = 'foobar'
key_name = 'lock:' + name
orig = Lock(conn, name, expire=100, id=b"a")
orig.acquire()
pytest.raises(TypeError, Lock, conn, name, id=object())
lock = Lock(conn, name, id=b"a")
pytest.raises(AlreadyAcquired, lock.acquire)
lock.extend(100)
lock.release() # this works, note that this ain't the object that acquired the lock
pytest.raises(NotAcquired, orig.release) # and this fails because lock was released above
assert conn.ttl(key_name) == -2
def distributed_lock(lock_name, expire=10, auto_renewal=True, blocking=False):
"""Context manager to acquire a lock visible by all processes.
This lock is implemented through Redis in order to be globally visible.
Args:
lock_name (str): name of the lock to be acquired.
expire (int): duration in seconds after which the lock is released if
not renewed in the meantime.
auto_renewal (bool): if ``True``, the lock is automatically renewed as long
as the context manager is still active.
blocking (bool): if ``True``, wait for the lock to be released. If ``False``,
return immediately, raising :class:`DistributedLockError`.
It is recommended to set ``expire`` to a small value and
``auto_renewal=True``, which ensures the lock gets released quickly in case
the process is killed without limiting the time that can be spent holding
the lock.
Raises:
DistributedLockError: when ``blocking`` is set to ``False`` and the lock is already acquired.
"""
if not lock_name:
raise ValueError('Lock name not specified.')
redis_url = app.config.get('CACHE_REDIS_URL')
redis = StrictRedis.from_url(redis_url)
lock = Lock(redis, lock_name, expire=expire, auto_renewal=auto_renewal)
if lock.acquire(blocking=blocking):
try:
yield
finally:
lock.release()
else:
raise DistributedLockError('Cannot acquire lock for %s', lock_name)
def test_owner_id(redis_server):
conn = StrictRedis(unix_socket_path=UDS_PATH)
unique_identifier = b"foobar-identifier"
lock = Lock(conn, "foobar-tok", expire=TIMEOUT/4, id=unique_identifier)
lock_id = lock.id
assert lock_id == unique_identifier
lock.acquire(blocking=False)
assert lock.get_owner_id() == unique_identifier
lock.release()
def test_extend_another_instance(conn):
"""It is possible to extend a lock using another instance of Lock with the
same name.
"""
name = 'foobar'
key_name = 'lock:' + name
lock = Lock(conn, name, expire=100)
lock.acquire()
assert 0 <= conn.ttl(key_name) <= 100
another_lock = Lock(conn, name, id=lock.id)
another_lock.extend(1000)
assert conn.ttl(key_name) > 100
def test_extend_another_instance_different_id_fail(conn):
"""It is impossible to extend a lock using another instance of Lock with
the same name, but different id.
"""
name = 'foobar'
key_name = 'lock:' + name
lock = Lock(conn, name, expire=100)
lock.acquire()
assert 0 <= conn.ttl(key_name) <= 100
another_lock = Lock(conn, name)
with pytest.raises(NotAcquired):
another_lock.extend(1000)
assert conn.ttl(key_name) <= 100
assert lock.id != another_lock.id
def test_timeout_expire_with_renewal(conn):
with Lock(conn, "foobar", expire=1, auto_renewal=True):
lock = Lock(conn, "foobar")
assert lock.acquire(timeout=2) == False
