def test_no_previous_owner_bypasses_watch(self):
"""
Coverage test. Internally the lock algorithm checks and sets a
watch on the nearest candidate node. If the node has been removed
between the time between the get_children and exists call, the we
immediately reattempt to get the lock without waiting on the watch.
"""
client = yield self.open_client()
path = yield client.create("/lock-no-previous")
# setup the client to create the intended environment
mock_client = self.mocker.patch(client)
mock_client.create(ANY, flags=ANY)
self.mocker.result(succeed("%s/%s" % (path, "lock-3")))
mock_client.get_children(path)
self.mocker.result(succeed(["lock-2", "lock-3"]))
mock_client.exists_and_watch("%s/%s" % (path, "lock-2"))
watch = Deferred()
self.mocker.result((succeed(False), watch))
mock_client.get_children(path)
self.mocker.result(succeed(["lock-3"]))
self.mocker.replay()
lock = Lock(path, mock_client)
yield lock.acquire()
self.assertTrue(lock.acquired)
def test_error_when_releasing_unacquired(self):
"""
If an attempt is made to release a lock, that not currently being held,
than a C{LockError} exception is raised.
"""
client = yield self.open_client()
lock_dir = yield client.create("/lock-multi-test")
lock = Lock(lock_dir, client)
self.failUnlessFailure(lock.release(), LockError)
def test_error_on_double_acquire(self):
"""
Attempting to acquire an already held lock, raises a Value Error.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
yield self.failUnlessFailure(lock.acquire(), LockError)
def test_acquire_release(self):
"""
A lock can be acquired and released.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
released = yield lock.release()
self.assertEqual(released, True)
def test_acquire_after_error(self):
"""
Any instance state associated with a failed acquired should be cleared
on error, allowing subsequent to succeed.
"""
client = yield self.open_client()
path = "/lock-test-acquire-after-error"
lock = Lock(path, client)
d = lock.acquire()
self.failUnlessFailure(d, NoNodeException)
yield d
yield client.create(path)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
def test_lock_reuse(self):
"""
A lock instance may be reused after an acquire/release cycle.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertTrue(lock.acquired)
yield lock.release()
self.assertFalse(lock.acquired)
yield lock.acquire()
self.assertTrue(lock.acquired)
yield lock.release()
self.assertFalse(lock.acquired)
def test_error_on_acquire_acquiring(self):
"""
Attempting to acquire the lock while an attempt is already in progress,
raises a LockError.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
# setup the client to create the intended environment
mock_client = self.mocker.patch(client)
mock_client.create(ANY, flags=ANY)
self.mocker.result(succeed("%s/%s" % (path, "lock-3")))
mock_client.get_children("/lock-test")
self.mocker.result(succeed(["lock-2", "lock-3"]))
mock_client.exists_and_watch("%s/%s" % (path, "lock-2"))
watch = Deferred()
self.mocker.result((succeed(True), watch))
self.mocker.replay()
# now we attempt to acquire the lock, rigged above to not succeed
d = lock.acquire()
test_deferred = Deferred()
# and next we schedule a lock attempt, which should fail as we're
# still attempting to acquire the lock.
def attempt_acquire():
# make sure lock was previously attempted acquired without
# error (disregarding that it was rigged to *fail*)
from twisted.python.failure import Failure
self.assertFalse(isinstance(d.result, Failure))
# acquire lock and expect to fail
self.failUnlessFailure(lock.acquire(), LockError)
# after we've verified the error handling, end the test
test_deferred.callback(None)
from twisted.internet import reactor
reactor.callLater(0.1, attempt_acquire)
yield test_deferred
def test_multiple_acquiring_clients(self):
"""
Multiple clients can compete for the lock, only one client's Lock
instance may hold the lock at any given moment.
"""
client = yield self.open_client()
client2 = yield self.open_client()
lock_dir = yield client.create("/lock-multi-test")
lock = Lock(lock_dir, client)
lock2 = Lock(lock_dir, client2)
yield lock.acquire()
self.assertTrue(lock.acquired)
lock2_acquire = lock2.acquire()
yield lock.release()
yield lock2_acquire
self.assertTrue(lock2.acquired)
self.assertFalse(lock.acquired)
def __init__(self, path, client, acl=None, persistent=False):
super(SerializedQueue, self).__init__(path, client, acl, persistent)
self._lock = Lock("%s/%s" % (self.path, "_lock"), client)
class SerializedQueue(Queue):
"""
A serialized queue ensures even with multiple consumers items are retrieved
and processed in the order they where placed in the queue.
This implementation aggregates a reliable queue, with a lock to provide
for serialized consumer access. The lock is released only when a queue item
has been processed.
"""
def __init__(self, path, client, acl=None, persistent=False):
super(SerializedQueue, self).__init__(path, client, acl, persistent)
self._lock = Lock("%s/%s" % (self.path, "_lock"), client)
def _item_processed_callback(self, result_code, item_path):
return self._lock.release()
def _filter_children(self, children, suffix="-processing"):
"""
Filter the lock from consideration as an item to be processed.
"""
children.sort()
for name in list(children):
if name.startswith('_'):
children.remove(name)
def _on_lock_directory_does_not_exist(self, failure):
"""
If the lock directory does not exist, go ahead and create it and
attempt to acquire the lock.
"""
failure.trap(zookeeper.NoNodeException)
d = self._client.create(self._lock.path)
d.addBoth(self._on_lock_created_or_exists)
return d
def _on_lock_created_or_exists(self, failure):
"""
The lock node creation will either result in success or node exists
error, if a concurrent client created the node first. In either case
we proceed with attempting to acquire the lock.
"""
if isinstance(failure, Failure):
failure.trap(zookeeper.NodeExistsException)
d = self._lock.acquire()
return d
def _on_lock_acquired(self, lock):
"""
After the exclusive queue lock is acquired, we proceed with an attempt
to fetch an item from the queue.
"""
d = super(SerializedQueue, self).get()
return d
def get(self):
"""
Get and remove an item from the queue. If no item is available
at the moment, a deferred is return that will fire when an item
is available.
"""
d = self._lock.acquire()
d.addErrback(self._on_lock_directory_does_not_exist)
d.addCallback(self._on_lock_acquired)
return d
def _get_item(self, children, request):
def fetch_node(name):
path = "/".join((self._path, name))
d = self._client.get(path)
d.addCallback(on_node_retrieved, path)
d.addErrback(on_reservation_failed)
return d
def on_node_retrieved((data, stat), path):
request.processing_children = False
request.callback(
QueueItem(
path, data, self._client, self._item_processed_callback))
def on_reservation_failed(failure=None):
"""If we can't get the node or reserve, continue processing
the children."""
if failure and not failure.check(
zookeeper.NodeExistsException, zookeeper.NoNodeException):
request.processing_children = True
request.errback(failure)
return
if children:
name = children.pop(0)
return fetch_node(name)
# If a watch fired while processing children, process it
# after the children list is exhausted.
request.processing_children = False
if request.refetch_children:
request.refetch_children = False
return self._get(request)
self._filter_children(children)
if not children:
return on_reservation_failed()
name = children.pop(0)
return fetch_node(name)