def test_simple_lock():
"""
Test a RedLock can be acquired.
"""
lock = RedLock("test_simple_lock", [{"host": "localhost"}], ttl=1000)
locked = lock.acquire()
lock.release()
assert locked is True
def test_from_url():
"""
Test a RedLock can be acquired via from_url.
"""
lock = RedLock("test_from_url", [{"url": "redis://localhost/0"}], ttl=1000)
locked = lock.acquire()
lock.release()
assert locked is True
def get_lock(lock):
"""
Get a distributed lock based on redis using RedLock algorithm
"""
return RedLock(lock, connection_details=[
{
'connection_pool': pool
},
])
def test_context_manager():
"""
Test a RedLock can be released by the context manager automically.
"""
with RedLock("test_context_manager", [{"host": "localhost"}], ttl=1000):
lock = RedLock("test_context_manager", [{
"host": "localhost"
}],
ttl=1000)
locked = lock.acquire()
assert locked == False
lock = RedLock("test_context_manager", [{"host": "localhost"}], ttl=1000)
locked = lock.acquire()
assert locked == True
lock.release()
def on_spawn_beeper(self, data):
with RedLock("redlock:{}".format(data["game_id"])):
map = ImpactMap()
map_data = self.redis.get(data["game_id"])
map.load(map_data)
beeper = map.spawn_beeper()
self.redis.set(data["game_id"], json.dumps(map.impact_map))
msg = {"handle": "common", "command": "spawnBeeper",
"params": {"x": beeper["x"], "y": beeper["y"]}}
emit("command", json.dumps(msg), room=data["game_id"])
def test_lock_with_validity():
"""
Test a RedLock can be acquired and the lock validity is also retruned.
"""
ttl = 1000
lock = RedLock("test_simple_lock", [{"host": "localhost"}], ttl=ttl)
locked, validity = lock.acquire_with_validity()
lock.release()
assert locked is True
assert 0 < validity < ttl - ttl * CLOCK_DRIFT_FACTOR - 2
def test_lock_with_multi_backend():
"""
Test a RedLock can be acquired when at least N/2+1 redis instances are alive.
Set redis instance with port 6380 down or debug sleep during test.
"""
lock = RedLock("test_simple_lock", connection_details=[
{"host": "localhost", "port": 6379, "db": 0, "socket_timeout": 0.2},
{"host": "localhost", "port": 6379, "db": 1, "socket_timeout": 0.2},
{"host": "localhost", "port": 6380, "db": 0, "socket_timeout": 0.2}], ttl=1000)
locked = lock.acquire()
lock.release()
assert locked == True
def test_lock_is_locked():
lock = RedLock("test_lock_is_locked")
# Clear possible initial states
[node.delete(lock.resource) for node in lock.redis_nodes]
assert lock.locked() is False
lock.acquire()
assert lock.locked() is True
lock.release()
assert lock.locked() is False
def _acquire_lock(self):
with RedLock(self._redlock_name,
connection_details=[self.conn],
retry_times=30) as _lock:
res = self.conn.get(self.key)
if res:
res = int(res)
if res + self.acquire_num > self.concurrent_num: # locked
return False
self.conn.incr(self.key, self.acquire_num)
self.conn.expire(self.key, self.expire) # 防止死锁
return True
def wrapper(*args):
conn = [{
'host': Config.MASTER_CACHE_HOST,
'port': Config.MASTER_CACHE_PORT,
'password': Config.MASTER_CACHE_PASSWORD,
'db': 0
}]
k = str(args[0].__class__) + func.__name__
v = cachesrv.get(k)
if not v:
with RedLock(k + '.lock', connection_details=conn):
v = json.dumps(func(*args))
cachesrv.set(k, v, Config.MASTER_CACHE_TTL)
return v
def test_context_manager():
"""
Test a RedLock can be released by the context manager automically.
"""
with RedLock("test_context_manager", [{"host": "localhost"}], ttl=1000):
lock = RedLock("test_context_manager", [{"host": "localhost"}], ttl=1000)
locked = lock.acquire()
assert locked == False
lock = RedLock("test_context_manager", [{"host": "localhost"}], ttl=1000)
locked = lock.acquire()
assert locked == True
# try to lock again within a with block
try:
with RedLock("test_context_manager", [{"host": "localhost"}]):
# shouldn't be allowed since someone has the lock already
assert False
except RedLockError:
# we expect this call to error out
pass
lock.release()
def finish_job():
print("Checking cq_jobs in 'waiting' state")
with RedLock("lock/recomputation_jobs/run_job"):
jobs = ContinuousQueryRecomputeJob.query.filter(
ContinuousQueryRecomputeJob.state == "waiting").all()
for recomputation_job in jobs:
time_interval_start_ts = float(
time.mktime(recomputation_job.time_interval_start.timetuple()))
last_run_start_ts = float(
time.mktime(recomputation_job.last_run_start.timetuple()))
if last_run_start_ts <= time_interval_start_ts:
recomputation_job.finish()
db.session.add(recomputation_job)
db.session.commit()
db.session.remove()
pass
async def _group_add_result(self, graph_uuid, group_uuid, task_uuid,
result):
with RedLock('%s_%s' % (graph_uuid, group_uuid),
connection_details=[{
'url': self._url
}]):
uuids_key = GRAPH_TASK_GROUP_UUIDS_PATTERN.format(
graph_uuid=graph_uuid, group_uuid=group_uuid)
results_key = GRAPH_TASK_GROUP_RESULT_PATTERN.format(
graph_uuid=graph_uuid, group_uuid=group_uuid)
pipe = self._connection.pipeline()
pipe.rpush(uuids_key, task_uuid)
pipe.rpush(results_key, json.dumps(result))
pipe.expire(uuids_key, self.app.config.backend_results_timelife)
pipe.expire(results_key, self.app.config.backend_results_timelife)
group_len = pipe.llen(uuids_key)
await pipe.execute()
return await group_len
def run_job():
print("Checking cq_jobs in 'created' or 'waiting' state")
with RedLock("lock/recomputation_jobs/run_job"):
jobs = ContinuousQueryRecomputeJob.query.filter(
or_(ContinuousQueryRecomputeJob.state == "created",
ContinuousQueryRecomputeJob.state == "waiting")).all()
for recomputation_job in jobs:
if recomputation_job.state == "created":
recomputation_job.last_run_start = recomputation_job.time_interval_end
recomputation_job.last_run_end = recomputation_job.time_interval_end
time_interval_start_ts = float(
time.mktime(recomputation_job.time_interval_start.timetuple()))
last_run_start_ts = float(
time.mktime(recomputation_job.last_run_start.timetuple()))
if last_run_start_ts > time_interval_start_ts:
if recomputation_job.last_execution_time:
interval_processed = recomputation_job.last_run_end.timestamp(
) - recomputation_job.last_run_start.timestamp()
time_delta = interval_processed * (
MAX_EXECUTION_TIME_PER_RUN /
recomputation_job.last_execution_time)
else:
time_delta = 12 * 3600
time_delta = min(time_delta, MAX_TIME_DELTA)
recomputation_job.last_run_end = recomputation_job.last_run_start
recomputation_job.last_run_start = recomputation_job.last_run_end - datetime.timedelta(
seconds=time_delta)
if time_delta == 0.0:
recomputation_job.finish()
else:
recomputation_job.run()
db.session.add(recomputation_job)
db.session.commit()
db.session.remove()
def get_lock(lockname, timeout=DEFAULT_TIMEOUT, retry_times=1, retry_delay=200):
try:
from django_redis import get_redis_connection
except ImportError:
raise Exception("Can't get default Redis connection")
else:
redis_client = get_redis_connection()
if timeout is None:
# redlock doesn't support infinite timeout
# anyway it's bad idea to have infinite lock
timeout = DEFAULT_TIMEOUT
lock = RedLock(lockname, [redis_client], retry_times=retry_times, retry_delay=retry_delay, ttl=int(timeout * 1000))
got_lock = lock.acquire()
try:
yield got_lock
finally:
if got_lock:
lock.release()
def acquire(self):
logger.debug("Acquiring global lock %s", self._lock_name)
try:
self._redlock = RedLock(self._lock_name,
connection_details=[self._redis_info],
ttl=self._lock_ttl)
acquired = self._redlock.acquire()
if not acquired:
logger.debug("Was unable to not acquire lock %s",
self._lock_name)
return False
logger.debug("Acquired lock %s", self._lock_name)
return True
except RedLockError:
logger.debug("Could not acquire lock %s", self._lock_name)
return False
except RedisError as re:
logger.debug("Could not connect to Redis for lock %s: %s",
self._lock_name, re)
return False
def get_long_lock(lockname, retry_times=1, retry_delay=200):
try:
from django_redis import get_redis_connection
except ImportError:
raise Exception("Can't get default Redis connection")
else:
redis_client = get_redis_connection()
ttl = int(long_lock_ttl * 1000)
lock = RedLock(lockname, [redis_client], retry_times=retry_times, retry_delay=retry_delay, ttl=ttl)
got_lock = lock.acquire()
if got_lock:
thread = LongLockUpdateThread(lock)
thread.start()
try:
yield got_lock
finally:
if got_lock:
lock.release()
thread.stop()
def test_default_connection_details_value():
"""
Test that RedLock instance could be created with
default value of `connection_details` argument.
"""
lock = RedLock("test_simple_lock")
def lock(self):
return RedLock("lock:wagtail_localize_pontoon.sync",
connection_details=[{
'url': settings.REDIS_URL,
}])
def _lockRead(self, keyspace): # test
with RedLock(keyspace[:2]):
return self._read(keyspace)
def test_redlock():
from redlock import RedLock
with RedLock("distributed_lock"):
pass
# them. About the only thing you can't do is ignore them.
# Because they change things. They push the human race forward.
# And while some may see them as the creazy ones, we see genius.
# Because the poeple who are crazy enough to think thay can change
# the world, are the ones who do."
#
# Created by Chyi Yaqing on 03/16/19 20:12.
# Copyright © 2019. Chyi Yaqing. All rights reserved.
#
# Distributed under terms of the MIT
"""
Basic Usage
"""
import os
import time
from redlock import RedLock
# get the string environment
env_dist = os.environ
redis_password = env_dist.get('REMOTE_REDIS_PASSWORD')
with RedLock("distributed_lock",
connection_details=[
# master 1
{'host': '192.168.31.156', 'port': 6379, 'db': 0, 'password': redis_password},
# master 2
{'host': '192.168.31.39', 'port': 6379, 'db': 0, 'password': redis_password},
]):
time.sleep(10)
print('Hello World')
def test_lock_release_return():
test_lock = RedLock("test_lock_release_return")
test_lock.acquire()
assert test_lock.release() == True
if type == 'init':
return init_process(command)
elif type == 'save':
return save_process(command)
elif type == 'load':
return load_process(command)
elif type == 'remit':
return remit_process(command)
else:
return end_process()
lock = RedLock("distributed_lock",
connection_details=[
{
'host': '192.168.111.128',
'port': 6379,
'db': 0
},
])
r = redis.Redis(host='192.168.111.128', port=6379, db=0)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('0.0.0.0', 8888))
s.listen(1)
while (1):
c, ip = s.accept()
print(ip)
while 1:
command = c.recv(1024)
if command:
lock.acquire()
ret = process_command(command).encode('utf-8')
ap.add_argument("--update-interval", type=int, default=60)
ap.add_argument("-v", "--verbose", action="store_true")
ap.add_argument("--debug", action="store_true")
ap.add_argument("--dry-run", action="store_true")
args = ap.parse_args()
if args.verbose:
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s")
if args.debug:
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)-8s %(message)s")
mc = pm.MongoClient(host=args.mongodb_uri, connect=False)
rc = redis.Redis(host=args.redis_host, port=args.redis_port, password=args.redis_pwd, db=0)
df = redis.Redis(host=args.redis_host, port=args.redis_port, password=args.redis_pwd, db=1)
lock = RedLock([{"host": args.redis_host, "port": args.redis_port, "db": 3}])
logging.info("building filter ...")
with mc.rssnews.news.find({}, {"uuid": True}) as cursor:
for news_item in cursor:
if df.scard(news_item["uuid"]) == 0:
for sym in news_item["symbols"]:
df.sadd(news_item["uuid"], sym)
logging.info("%d existing urls in total.", df.dbsize())
logging.info("generating tasks ...")
with mc.rssnews.feed.find() as cursor:
logging.info("number of rss feeds = %d", cursor.count())
tasks = []
for item in cursor:
logging.debug("rss=%(url)s", item)
def __init__(self, key, ttl=1000, retry=None, delay=None):
self.key = key
self.ttl = ttl
retry = retry if retry else settings.REDIS_BLOCKER_RETRY
delay = delay if delay else settings.REDIS_BLOCKER_DELAY
self.redis = RedLock(key, settings.REDIS_BLOCKER, retry, delay, ttl)
def lock(self):
return RedLock("lock:wagtail_localize_pontoon.sync",
connection_details=[{
'url':
settings.REDIS_URL or 'redis://localhost:6379/0',
}])
def _lockWrite(self, keyspace, valuespace): # test
with RedLock(keyspace[:2]):
return self._write(keyspace, valuespace)
'port': 6379,
'db':0
},
{
'host':'localhost',
'port': 6479,
'db':0
},
{
'host':'localhost',
'port': 6579,
'db':0
},
{
'host':'localhost',
'port': 6679,
'db':0
}
]
lock_key = 'user-lck-laoqian'
lock = RedLock(lock_key)
lock.acquire()
print("i am locked")
lock.release()
