def test_it_should_be_able_to_be_acquired_and_released(self):
self.lock = Lock(self.client,
"locks/foo/bar/%d" % random.uniform(1, 100000))
self.lock.acquire()
node = self.client.get(self.lock.key)
expect(node.value).to.eq(self.lock.token)
expect(node.ttl).to.eq(60)
self.lock.release()
expect(self.client.get(self.lock.key).value).to.eq("0")
def setUp(self):
self.client = etcd.Client()
self.key = "lock-%d" % random.uniform(1, 100000)
self.locks = [
Lock(etcd.Client(), self.key, ttl=2, renewSecondsPrior=1),
Lock(etcd.Client(), self.key, ttl=5, renewSecondsPrior=1),
Lock(etcd.Client(),
self.key,
ttl=2,
renewSecondsPrior=None,
timeout=2)
]
def test_it_should_be_able_to_be_acquired_and_released(self):
self.lock = Lock(self.client, "locks/foo/bar/%d" % random.uniform(1,100000))
self.lock.acquire()
node = self.client.get(self.lock.key)
expect(node.value).to.eq(self.lock.token)
expect(node.ttl).to.eq(60)
self.lock.release()
expect(self.client.get(self.lock.key).value).to.eq("0")
def polling_tasks(self):
"""
以轮询的方式, 获得将要到期的任务
:return:
"""
coll = mongo_viae.ViaeTask
lock_name = 'mongo-polling'
lock_key = '%s/%s' % (settings.ETCD_LOCK_PREFIX, lock_name)
ttl = getattr(settings, 'ETCD_LOCK_TTL') if hasattr(settings, 'ETCD_LOCK_TTL') else 10
# 分布式锁
lock = Lock(etcd_client, lock_key, ttl=ttl)
# 是否获取到了锁
acquired = False
# 尝试次数
attempt_count = 0
# 最多尝试3次
while not acquired and attempt_count < 3:
attempt_count += 1
try:
acquired = lock.acquire(timeout=10)
if not acquired:
continue
# 以TASK_ETA_THRESHOLD为限, 所有即将开始执行的task, 都转移到celery queue里面
tasks = coll.find({'eta': {'$lt': datetime.utcnow() + timedelta(seconds=settings.TASK_ETA_THRESHOLD)}})
for task in tasks:
args = task.get('args', [])
kwargs = task.get('kwargs', {})
routing_key = task.get('routing_key')
eta = task.get('eta')
expires = task.get('expires')
task_id = task.get('_id')
task = task['task']
app.send_task(task, serializer='json', args=args, kwargs=kwargs, eta=eta,
routing_key=routing_key, expires=expires)
coll.remove({'_id': task_id})
finally:
if acquired:
lock.release()
def setUp(self):
self.client = etcd.Client()
self.lock = Lock(self.client, "lock-%d" % random.uniform(1,100000), ttl=2, renewSecondsPrior=1)
class SingleLockTests(unittest.TestCase):
def setUp(self):
self.client = etcd.Client()
self.lock = Lock(self.client, "lock-%d" % random.uniform(1,100000), ttl=2, renewSecondsPrior=1)
def teardown(self):
if (self.lock != None and self.lock.token != None):
self.lock.release()
def test_it_should_require_an_etcd_client(self):
expect(lambda: Lock(None, 'lock/foo')).to.raise_error(ValueError)
def test_it_should_require_a_valid_ttl(self):
expect(lambda: Lock(self.client, 'lock/foo', ttl=0)).to.raise_error(ValueError)
def test_it_should_require_a_valid_renew_prior_time(self):
expect(lambda: Lock(self.client, 'lock/foo', renewSecondsPrior=-5)).to.raise_error(ValueError)
def test_it_should_require_renew_prior_to_be_less_than_the_ttl(self):
expect(lambda: Lock(self.client, 'lock/foo', ttl=20, renewSecondsPrior=20)).to.raise_error(ValueError)
def test_it_should_require_an_etcd_key(self):
expect(lambda: Lock(self.client, key=None)).to.raise_error(ValueError)
def test_it_should_be_able_to_be_acquired_and_released(self):
self.lock = Lock(self.client, "locks/foo/bar/%d" % random.uniform(1,100000))
self.lock.acquire()
node = self.client.get(self.lock.key)
expect(node.value).to.eq(self.lock.token)
expect(node.ttl).to.eq(60)
self.lock.release()
expect(self.client.get(self.lock.key).value).to.eq("0")
def test_it_should_renew_the_lock_while_still_being_used(self):
self.lock.acquire()
time.sleep(2)
expect(self.client.get(self.lock.key).value).to.eq(self.lock.token)
def test_it_cancel_the_renew_after_releasing(self):
self.lock.acquire()
time.sleep(1)
self.lock.release()
time.sleep(3)
expect(self.client.get(self.lock.key).value).to.eq("0")
def test_it_should_not_renew_if_renewSecondsPrior_is_None(self):
self.lock.renewSecondsPrior = None
self.lock.acquire()
time.sleep(3)
expect(lambda: self.client.get(self.lock.key)).to.raise_error(etcd.EtcdKeyNotFound)
def test_it_should_be_usable_as_a_context_manager(self):
with self.lock as locked:
expect(locked).to.equal(True)
expect(self.client.get(self.lock.key).value).to.eq(self.lock.token)
expect(self.client.get(self.lock.key).value).to.eq("0")
def test_it_should_require_renew_prior_to_be_less_than_the_ttl(self):
expect(
lambda: Lock(self.client, 'lock/foo', ttl=20, renewSecondsPrior=20
)).to.raise_error(ValueError)
def test_it_should_require_an_etcd_key(self):
expect(lambda: Lock(self.client, key=None)).to.raise_error(ValueError)
def test_it_should_require_a_valid_renew_prior_time(self):
expect(lambda: Lock(self.client, 'lock/foo', renewSecondsPrior=-5)
).to.raise_error(ValueError)
def test_it_should_require_a_valid_ttl(self):
expect(lambda: Lock(self.client, 'lock/foo', ttl=0)).to.raise_error(
ValueError)
def test_it_should_require_an_etcd_client(self):
expect(lambda: Lock(None, 'lock/foo')).to.raise_error(ValueError)
def setUp(self):
self.client = etcd.Client()
self.lock = Lock(self.client,
"lock-%d" % random.uniform(1, 100000),
ttl=2,
renewSecondsPrior=1)
class SingleLockTests(unittest.TestCase):
def setUp(self):
self.client = etcd.Client()
self.lock = Lock(self.client,
"lock-%d" % random.uniform(1, 100000),
ttl=2,
renewSecondsPrior=1)
def teardown(self):
if (self.lock != None and self.lock.token != None):
self.lock.release()
def test_it_should_require_an_etcd_client(self):
expect(lambda: Lock(None, 'lock/foo')).to.raise_error(ValueError)
def test_it_should_require_a_valid_ttl(self):
expect(lambda: Lock(self.client, 'lock/foo', ttl=0)).to.raise_error(
ValueError)
def test_it_should_require_a_valid_renew_prior_time(self):
expect(lambda: Lock(self.client, 'lock/foo', renewSecondsPrior=-5)
).to.raise_error(ValueError)
def test_it_should_require_renew_prior_to_be_less_than_the_ttl(self):
expect(
lambda: Lock(self.client, 'lock/foo', ttl=20, renewSecondsPrior=20
)).to.raise_error(ValueError)
def test_it_should_require_an_etcd_key(self):
expect(lambda: Lock(self.client, key=None)).to.raise_error(ValueError)
def test_it_should_be_able_to_be_acquired_and_released(self):
self.lock = Lock(self.client,
"locks/foo/bar/%d" % random.uniform(1, 100000))
self.lock.acquire()
node = self.client.get(self.lock.key)
expect(node.value).to.eq(self.lock.token)
expect(node.ttl).to.eq(60)
self.lock.release()
expect(self.client.get(self.lock.key).value).to.eq("0")
def test_it_should_renew_the_lock_while_still_being_used(self):
self.lock.acquire()
time.sleep(2)
expect(self.client.get(self.lock.key).value).to.eq(self.lock.token)
def test_it_cancel_the_renew_after_releasing(self):
self.lock.acquire()
time.sleep(1)
self.lock.release()
time.sleep(3)
expect(self.client.get(self.lock.key).value).to.eq("0")
def test_it_should_not_renew_if_renewSecondsPrior_is_None(self):
self.lock.renewSecondsPrior = None
self.lock.acquire()
time.sleep(3)
expect(lambda: self.client.get(self.lock.key)).to.raise_error(
etcd.EtcdKeyNotFound)
def test_it_should_be_usable_as_a_context_manager(self):
with self.lock as locked:
expect(locked).to.equal(True)
expect(self.client.get(self.lock.key).value).to.eq(self.lock.token)
expect(self.client.get(self.lock.key).value).to.eq("0")
