Esempio n. 1
0
    def test_no_previous_owner_bypasses_watch(self):
        """
        Coverage test.  Internally the lock algorithm checks and sets a
        watch on the nearest candidate node. If the node has been removed
        between the time between the get_children and exists call, the we
        immediately reattempt to get the lock without waiting on the watch.
        """
        client = yield self.open_client()
        path = yield client.create("/lock-no-previous")

        # setup the client to create the intended environment
        mock_client = self.mocker.patch(client)
        mock_client.create(ANY, flags=ANY)
        self.mocker.result(succeed("%s/%s" % (path, "lock-3")))

        mock_client.get_children(path)
        self.mocker.result(succeed(["lock-2", "lock-3"]))

        mock_client.exists_and_watch("%s/%s" % (path, "lock-2"))
        watch = Deferred()
        self.mocker.result((succeed(False), watch))

        mock_client.get_children(path)
        self.mocker.result(succeed(["lock-3"]))
        self.mocker.replay()

        lock = Lock(path, mock_client)
        yield lock.acquire()
        self.assertTrue(lock.acquired)
Esempio n. 2
0
 def test_error_when_releasing_unacquired(self):
     """
     If an attempt is made to release a lock, that not currently being held,
     than a C{LockError} exception is raised.
     """
     client = yield self.open_client()
     lock_dir = yield client.create("/lock-multi-test")
     lock = Lock(lock_dir, client)
     self.failUnlessFailure(lock.release(), LockError)
Esempio n. 3
0
 def test_error_on_double_acquire(self):
     """
     Attempting to acquire an already held lock, raises a Value Error.
     """
     client = yield self.open_client()
     path = yield client.create("/lock-test")
     lock = Lock(path, client)
     yield lock.acquire()
     self.assertEqual(lock.acquired, True)
     yield self.failUnlessFailure(lock.acquire(), LockError)
Esempio n. 4
0
 def test_acquire_release(self):
     """
     A lock can be acquired and released.
     """
     client = yield self.open_client()
     path = yield client.create("/lock-test")
     lock = Lock(path, client)
     yield lock.acquire()
     self.assertEqual(lock.acquired, True)
     released = yield lock.release()
     self.assertEqual(released, True)
Esempio n. 5
0
 def test_acquire_after_error(self):
     """
     Any instance state associated with a failed acquired should be cleared
     on error, allowing subsequent to succeed.
     """
     client = yield self.open_client()
     path = "/lock-test-acquire-after-error"
     lock = Lock(path, client)
     d = lock.acquire()
     self.failUnlessFailure(d, NoNodeException)
     yield d
     yield client.create(path)
     yield lock.acquire()
     self.assertEqual(lock.acquired, True)
Esempio n. 6
0
 def test_lock_reuse(self):
     """
     A lock instance may be reused after an acquire/release cycle.
     """
     client = yield self.open_client()
     path = yield client.create("/lock-test")
     lock = Lock(path, client)
     yield lock.acquire()
     self.assertTrue(lock.acquired)
     yield lock.release()
     self.assertFalse(lock.acquired)
     yield lock.acquire()
     self.assertTrue(lock.acquired)
     yield lock.release()
     self.assertFalse(lock.acquired)
Esempio n. 7
0
    def test_error_on_acquire_acquiring(self):
        """
        Attempting to acquire the lock while an attempt is already in progress,
        raises a LockError.
        """
        client = yield self.open_client()
        path = yield client.create("/lock-test")
        lock = Lock(path, client)

        # setup the client to create the intended environment
        mock_client = self.mocker.patch(client)
        mock_client.create(ANY, flags=ANY)
        self.mocker.result(succeed("%s/%s" % (path, "lock-3")))

        mock_client.get_children("/lock-test")
        self.mocker.result(succeed(["lock-2", "lock-3"]))

        mock_client.exists_and_watch("%s/%s" % (path, "lock-2"))
        watch = Deferred()
        self.mocker.result((succeed(True), watch))
        self.mocker.replay()

        # now we attempt to acquire the lock, rigged above to not succeed
        d = lock.acquire()
        test_deferred = Deferred()

        # and next we schedule a lock attempt, which should fail as we're
        # still attempting to acquire the lock.
        def attempt_acquire():
            # make sure lock was previously attempted acquired without
            # error (disregarding that it was rigged to *fail*)
            from twisted.python.failure import Failure
            self.assertFalse(isinstance(d.result, Failure))

            # acquire lock and expect to fail
            self.failUnlessFailure(lock.acquire(), LockError)

            # after we've verified the error handling, end the test
            test_deferred.callback(None)

        from twisted.internet import reactor
        reactor.callLater(0.1, attempt_acquire)

        yield test_deferred
Esempio n. 8
0
    def test_multiple_acquiring_clients(self):
        """
        Multiple clients can compete for the lock, only one client's Lock
        instance may hold the lock at any given moment.
        """
        client = yield self.open_client()
        client2 = yield self.open_client()
        lock_dir = yield client.create("/lock-multi-test")

        lock = Lock(lock_dir, client)
        lock2 = Lock(lock_dir, client2)

        yield lock.acquire()
        self.assertTrue(lock.acquired)
        lock2_acquire = lock2.acquire()
        yield lock.release()
        yield lock2_acquire
        self.assertTrue(lock2.acquired)
        self.assertFalse(lock.acquired)
Esempio n. 9
0
 def __init__(self, path, client, acl=None, persistent=False):
     super(SerializedQueue, self).__init__(path, client, acl, persistent)
     self._lock = Lock("%s/%s" % (self.path, "_lock"), client)
Esempio n. 10
0
class SerializedQueue(Queue):
    """
    A serialized queue ensures even with multiple consumers items are retrieved
    and processed in the order they where placed in the queue.

    This implementation aggregates a reliable queue, with a lock to provide
    for serialized consumer access. The lock is released only when a queue item
    has been processed.
    """

    def __init__(self, path, client, acl=None, persistent=False):
        super(SerializedQueue, self).__init__(path, client, acl, persistent)
        self._lock = Lock("%s/%s" % (self.path, "_lock"), client)

    def _item_processed_callback(self, result_code, item_path):
        return self._lock.release()

    def _filter_children(self, children, suffix="-processing"):
        """
        Filter the lock from consideration as an item to be processed.
        """
        children.sort()
        for name in list(children):
            if name.startswith('_'):
                children.remove(name)

    def _on_lock_directory_does_not_exist(self, failure):
        """
        If the lock directory does not exist, go ahead and create it and
        attempt to acquire the lock.
        """
        failure.trap(zookeeper.NoNodeException)
        d = self._client.create(self._lock.path)
        d.addBoth(self._on_lock_created_or_exists)
        return d

    def _on_lock_created_or_exists(self, failure):
        """
        The lock node creation will either result in success or node exists
        error, if a concurrent client created the node first. In either case
        we proceed with attempting to acquire the lock.
        """
        if isinstance(failure, Failure):
            failure.trap(zookeeper.NodeExistsException)
        d = self._lock.acquire()
        return d

    def _on_lock_acquired(self, lock):
        """
        After the exclusive queue lock is acquired, we proceed with an attempt
        to fetch an item from the queue.
        """
        d = super(SerializedQueue, self).get()
        return d

    def get(self):
        """
        Get and remove an item from the queue. If no item is available
        at the moment, a deferred is return that will fire when an item
        is available.
        """
        d = self._lock.acquire()

        d.addErrback(self._on_lock_directory_does_not_exist)
        d.addCallback(self._on_lock_acquired)
        return d

    def _get_item(self, children, request):

        def fetch_node(name):
            path = "/".join((self._path, name))
            d = self._client.get(path)
            d.addCallback(on_node_retrieved, path)
            d.addErrback(on_reservation_failed)
            return d

        def on_node_retrieved((data, stat), path):
            request.processing_children = False
            request.callback(
                QueueItem(
                    path, data, self._client, self._item_processed_callback))

        def on_reservation_failed(failure=None):
            """If we can't get the node or reserve, continue processing
            the children."""
            if failure and not failure.check(
                zookeeper.NodeExistsException, zookeeper.NoNodeException):
                request.processing_children = True
                request.errback(failure)
                return

            if children:
                name = children.pop(0)
                return fetch_node(name)

            # If a watch fired while processing children, process it
            # after the children list is exhausted.
            request.processing_children = False
            if request.refetch_children:
                request.refetch_children = False
                return self._get(request)

        self._filter_children(children)

        if not children:
            return on_reservation_failed()

        name = children.pop(0)
        return fetch_node(name)