def test_lock_reuse(self):
"""
A lock instance may be reused after an acquire/release cycle.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertTrue(lock.acquired)
yield lock.release()
self.assertFalse(lock.acquired)
yield lock.acquire()
self.assertTrue(lock.acquired)
yield lock.release()
self.assertFalse(lock.acquired)
def test_error_when_releasing_unacquired(self):
"""
If an attempt is made to release a lock, that not currently being held,
than a C{LockError} exception is raised.
"""
client = yield self.open_client()
lock_dir = yield client.create("/lock-multi-test")
lock = Lock(lock_dir, client)
self.failUnlessFailure(lock.release(), LockError)
def test_acquire_release(self):
"""
A lock can be acquired and released.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
released = yield lock.release()
self.assertEqual(released, True)
def test_multiple_acquiring_clients(self):
"""
Multiple clients can compete for the lock, only one client's Lock
instance may hold the lock at any given moment.
"""
client = yield self.open_client()
client2 = yield self.open_client()
lock_dir = yield client.create("/lock-multi-test")
lock = Lock(lock_dir, client)
lock2 = Lock(lock_dir, client2)
yield lock.acquire()
self.assertTrue(lock.acquired)
lock2_acquire = lock2.acquire()
yield lock.release()
yield lock2_acquire
self.assertTrue(lock2.acquired)
self.assertFalse(lock.acquired)
class SerializedQueue(Queue):
"""
A serialized queue ensures even with multiple consumers items are retrieved
and processed in the order they where placed in the queue.
This implementation aggregates a reliable queue, with a lock to provide
for serialized consumer access. The lock is released only when a queue item
has been processed.
"""
def __init__(self, path, client, acl=None, persistent=False):
super(SerializedQueue, self).__init__(path, client, acl, persistent)
self._lock = Lock("%s/%s" % (self.path, "_lock"), client)
def _item_processed_callback(self, result_code, item_path):
return self._lock.release()
def _filter_children(self, children, suffix="-processing"):
"""
Filter the lock from consideration as an item to be processed.
"""
children.sort()
for name in list(children):
if name.startswith('_'):
children.remove(name)
def _on_lock_directory_does_not_exist(self, failure):
"""
If the lock directory does not exist, go ahead and create it and
attempt to acquire the lock.
"""
failure.trap(zookeeper.NoNodeException)
d = self._client.create(self._lock.path)
d.addBoth(self._on_lock_created_or_exists)
return d
def _on_lock_created_or_exists(self, failure):
"""
The lock node creation will either result in success or node exists
error, if a concurrent client created the node first. In either case
we proceed with attempting to acquire the lock.
"""
if isinstance(failure, Failure):
failure.trap(zookeeper.NodeExistsException)
d = self._lock.acquire()
return d
def _on_lock_acquired(self, lock):
"""
After the exclusive queue lock is acquired, we proceed with an attempt
to fetch an item from the queue.
"""
d = super(SerializedQueue, self).get()
return d
def get(self):
"""
Get and remove an item from the queue. If no item is available
at the moment, a deferred is return that will fire when an item
is available.
"""
d = self._lock.acquire()
d.addErrback(self._on_lock_directory_does_not_exist)
d.addCallback(self._on_lock_acquired)
return d
def _get_item(self, children, request):
def fetch_node(name):
path = "/".join((self._path, name))
d = self._client.get(path)
d.addCallback(on_node_retrieved, path)
d.addErrback(on_reservation_failed)
return d
def on_node_retrieved((data, stat), path):
request.processing_children = False
request.callback(
QueueItem(
path, data, self._client, self._item_processed_callback))
def on_reservation_failed(failure=None):
"""If we can't get the node or reserve, continue processing
the children."""
if failure and not failure.check(
zookeeper.NodeExistsException, zookeeper.NoNodeException):
request.processing_children = True
request.errback(failure)
return
if children:
name = children.pop(0)
return fetch_node(name)
# If a watch fired while processing children, process it
# after the children list is exhausted.
request.processing_children = False
if request.refetch_children:
request.refetch_children = False
return self._get(request)
self._filter_children(children)
if not children:
return on_reservation_failed()
name = children.pop(0)
return fetch_node(name)