Exemple #1
0
    def test_lock_one(self):
        lock_name = uuid.uuid4().hex
        lock = self.client.Lock(self.lockpath, lock_name)
        event = threading.Event()

        thread = threading.Thread(target=self._thread_lock_acquire_til_event,
                                  args=(lock_name, lock, event))
        thread.start()

        lock2_name = uuid.uuid4().hex
        anotherlock = self.client.Lock(self.lockpath, lock2_name)

        # wait for any contender to show up on the lock
        wait(anotherlock.contenders)
        eq_(anotherlock.contenders(), [lock_name])

        with self.condition:
            while self.active_thread != lock_name:
                self.condition.wait()

        # release the lock
        event.set()

        with self.condition:
            while self.active_thread:
                self.condition.wait()
        self.released.wait()
        thread.join()
Exemple #2
0
    def test_lock_reconnect(self):
        event = threading.Event()
        other_lock = self.client.Lock(self.lockpath, 'contender')
        thread = threading.Thread(target=self._thread_lock_acquire_til_event,
                                  args=('contender', other_lock, event))

        # acquire the lock ourselves first to make the contender line up
        lock = self.client.Lock(self.lockpath, "test")
        lock.acquire()

        thread.start()
        # wait for the contender to line up on the lock
        wait(lambda: len(lock.contenders()) == 2)
        eq_(lock.contenders(), ['test', 'contender'])

        self.expire_session()

        lock.release()

        with self.condition:
            while not self.active_thread:
                self.condition.wait()
            eq_(self.active_thread, 'contender')

        event.set()
        thread.join()
Exemple #3
0
    def test_lock_reconnect(self):
        event = threading.Event()
        other_lock = self.client.Lock(self.lockpath, 'contender')
        thread = threading.Thread(target=self._thread_lock_acquire_til_event,
                                  args=('contender', other_lock, event))

        # acquire the lock ourselves first to make the contender line up
        lock = self.client.Lock(self.lockpath, "test")
        lock.acquire()

        thread.start()
        # wait for the contender to line up on the lock
        wait(lambda: len(lock.contenders()) == 2)
        eq_(lock.contenders(), ['test', 'contender'])

        self.expire_session()

        lock.release()

        with self.condition:
            while not self.active_thread:
                self.condition.wait()
            eq_(self.active_thread, 'contender')

        event.set()
        thread.join()
Exemple #4
0
    def test_lock_one(self):
        lock_name = uuid.uuid4().hex
        lock = self.client.Lock(self.lockpath, lock_name)
        event = threading.Event()

        thread = threading.Thread(target=self._thread_lock_acquire_til_event,
            args=(lock_name, lock, event))
        thread.start()

        lock2_name = uuid.uuid4().hex
        anotherlock = self.client.Lock(self.lockpath, lock2_name)

        # wait for any contender to show up on the lock
        wait(anotherlock.contenders)
        eq_(anotherlock.contenders(), [lock_name])

        with self.condition:
            while self.active_thread != lock_name:
                self.condition.wait()

        # release the lock
        event.set()

        with self.condition:
            while self.active_thread:
                self.condition.wait()
        self.released.wait()
Exemple #5
0
    def test_logging(self):
        handler = InstalledHandler('ZooKeeper')
        try:
            handle = zookeeper.init('zookeeper.example.com:2181')
            zookeeper.close(handle)
        except Exception:
            pass

        wait(lambda: [r for r in handler.records
                       if 'environment' in r.getMessage()]
             )
        handler.clear()
        kazoo.klog.setup_logging()

        # Test that the filter for the "Exceeded deadline by" noise works.
        # cheat and bypass zk by writing to the pipe directly.
        os.write(kazoo.klog._logging_pipe[1],
                 '2012-01-06 16:45:44,572:43673(0x1004f6000):ZOO_WARN@'
                 'zookeeper_interest@1461: Exceeded deadline by 27747ms\n')
        wait(lambda: [r for r in handler.records
                       if ('Exceeded deadline by' in r.getMessage()
                           and r.levelno == logging.DEBUG)
                       ]
             )

        self.assertFalse([r for r in handler.records
                          if ('Exceeded deadline by' in r.getMessage()
                              and r.levelno == logging.WARNING)
                          ])

        handler.uninstall()
Exemple #6
0
    def test_dirty_sock(self):
        client = self.client
        read_sock = client._connection._read_sock
        write_sock = client._connection._write_sock

        # add a stray byte to the socket and ensure that doesn't
        # blow up client. simulates case where some error leaves
        # a byte in the socket which doesn't correspond to the
        # request queue.
        write_sock.send(b'\0')

        # eventually this byte should disappear from socket
        wait(lambda: client.handler.select([read_sock], [], [], 0)[0] == [])
Exemple #7
0
    def test_dirty_sock(self):
        client = self.client
        read_sock = client._connection._read_sock
        write_sock = client._connection._write_sock

        # add a stray byte to the socket and ensure that doesn't
        # blow up client. simulates case where some error leaves
        # a byte in the socket which doesn't correspond to the
        # request queue.
        write_sock.send(b'\0')

        # eventually this byte should disappear from socket
        wait(lambda: client.handler.select([read_sock], [], [], 0)[0] == [])
Exemple #8
0
    def test_dirty_pipe(self):
        client = self.client
        read_pipe = client._connection._read_pipe
        write_pipe = client._connection._write_pipe

        # add a stray byte to the pipe and ensure that doesn't
        # blow up client. simulates case where some error leaves
        # a byte in the pipe which doesn't correspond to the
        # request queue.
        os.write(write_pipe, b'\0')

        # eventually this byte should disappear from pipe
        wait(lambda: client.handler.select([read_pipe], [], [], 0)[0] == [])
Exemple #9
0
    def test_dirty_pipe(self):
        client = self.client
        read_pipe = client._connection._read_pipe
        write_pipe = client._connection._write_pipe

        # add a stray byte to the pipe and ensure that doesn't
        # blow up client. simulates case where some error leaves
        # a byte in the pipe which doesn't correspond to the
        # request queue.
        os.write(write_pipe, b'\0')

        # eventually this byte should disappear from pipe
        wait(lambda: client.handler.select([read_pipe], [], [], 0)[0] == [])
Exemple #10
0
    def test_lock(self):
        threads = []
        names = ["contender" + str(i) for i in range(5)]

        contender_bits = {}

        for name in names:
            e = threading.Event()

            l = self.client.Lock(self.lockpath, name)
            t = threading.Thread(target=self._thread_lock_acquire_til_event,
                                 args=(name, l, e))
            contender_bits[name] = (t, e)
            threads.append(t)

        # acquire the lock ourselves first to make the others line up
        lock = self.client.Lock(self.lockpath, "test")
        lock.acquire()

        for t in threads:
            t.start()

        # wait for everyone to line up on the lock
        wait(lambda: len(lock.contenders()) == 6)
        contenders = lock.contenders()

        eq_(contenders[0], "test")
        contenders = contenders[1:]
        remaining = list(contenders)

        # release the lock and contenders should claim it in order
        lock.release()

        for contender in contenders:
            thread, event = contender_bits[contender]

            with self.condition:
                while not self.active_thread:
                    self.condition.wait()
                eq_(self.active_thread, contender)

            eq_(lock.contenders(), remaining)
            remaining = remaining[1:]

            event.set()

            with self.condition:
                while self.active_thread:
                    self.condition.wait()
        for thread in threads:
            thread.join()
Exemple #11
0
    def test_lock(self):
        threads = []
        names = ["contender" + str(i) for i in range(5)]

        contender_bits = {}

        for name in names:
            e = threading.Event()

            l = self.client.Lock(self.lockpath, name)
            t = threading.Thread(target=self._thread_lock_acquire_til_event,
                args=(name, l, e))
            contender_bits[name] = (t, e)
            threads.append(t)

        # acquire the lock ourselves first to make the others line up
        lock = self.client.Lock(self.lockpath, "test")
        lock.acquire()

        for t in threads:
            t.start()

        # wait for everyone to line up on the lock
        wait(lambda: len(lock.contenders()) == 6)
        contenders = lock.contenders()

        eq_(contenders[0], "test")
        contenders = contenders[1:]
        remaining = list(contenders)

        # release the lock and contenders should claim it in order
        lock.release()

        for contender in contenders:
            thread, event = contender_bits[contender]

            with self.condition:
                while not self.active_thread:
                    self.condition.wait()
                eq_(self.active_thread, contender)

            eq_(lock.contenders(), remaining)
            remaining = remaining[1:]

            event.set()

            with self.condition:
                while self.active_thread:
                    self.condition.wait()
        for thread in threads:
            thread.join()
Exemple #12
0
    def test_lock_cancel(self):
        event1 = threading.Event()
        lock1 = self.client.Lock(self.lockpath, "one")
        thread1 = threading.Thread(target=self._thread_lock_acquire_til_event,
                                   args=("one", lock1, event1))
        thread1.start()

        # wait for this thread to acquire the lock
        with self.condition:
            if not self.active_thread:
                self.condition.wait(5)
                eq_(self.active_thread, "one")

        client2 = self._get_client()
        client2.start()
        event2 = threading.Event()
        lock2 = client2.Lock(self.lockpath, "two")
        thread2 = threading.Thread(target=self._thread_lock_acquire_til_event,
                                   args=("two", lock2, event2))
        thread2.start()

        # this one should block in acquire. check that it is a contender
        wait(lambda: len(lock2.contenders()) > 1)
        eq_(lock2.contenders(), ["one", "two"])

        lock2.cancel()
        with self.condition:
            if "two" not in self.cancelled_threads:
                self.condition.wait()
                assert "two" in self.cancelled_threads

        eq_(lock2.contenders(), ["one"])

        thread2.join()
        event1.set()
        thread1.join()
        client2.stop()
Exemple #13
0
    def test_lock_cancel(self):
        event1 = threading.Event()
        lock1 = self.client.Lock(self.lockpath, "one")
        thread1 = threading.Thread(target=self._thread_lock_acquire_til_event,
            args=("one", lock1, event1))
        thread1.start()

        # wait for this thread to acquire the lock
        with self.condition:
            if not self.active_thread:
                self.condition.wait(5)
                eq_(self.active_thread, "one")

        client2 = self._get_client()
        client2.start()
        event2 = threading.Event()
        lock2 = client2.Lock(self.lockpath, "two")
        thread2 = threading.Thread(target=self._thread_lock_acquire_til_event,
            args=("two", lock2, event2))
        thread2.start()

        # this one should block in acquire. check that it is a contender
        wait(lambda: len(lock2.contenders()) > 1)
        eq_(lock2.contenders(), ["one", "two"])

        lock2.cancel()
        with self.condition:
            if not "two" in self.cancelled_threads:
                self.condition.wait()
                assert "two" in self.cancelled_threads

        eq_(lock2.contenders(), ["one"])

        thread2.join()
        event1.set()
        thread1.join()
        client2.stop()
Exemple #14
0
    def test_election(self):
        elections = {}
        threads = {}
        for _ in range(3):
            contender = "c" + uuid.uuid4().hex
            elections[contender] = self.client.Election(self.path, contender)
            threads[contender] = self._spawn_contender(contender,
                                                       elections[contender])

        # wait for a leader to be elected
        times = 0
        with self.condition:
            while not self.leader_id:
                self.condition.wait(5)
                times += 1
                if times > 5:
                    raise Exception("Still not a leader: lid: %s",
                                    self.leader_id)

        election = self.client.Election(self.path)

        # make sure all contenders are in the pool
        wait(lambda: len(election.contenders()) == len(elections))
        contenders = election.contenders()

        eq_(set(contenders), set(elections.keys()))

        # first one in list should be leader
        first_leader = contenders[0]
        eq_(first_leader, self.leader_id)

        # tell second one to cancel election. should never get elected.
        elections[contenders[1]].cancel()

        # make leader exit. third contender should be elected.
        self.exit_event.set()
        with self.condition:
            while self.leader_id == first_leader:
                self.condition.wait(45)
        eq_(self.leader_id, contenders[2])
        self._check_thread_error()

        # make first contender re-enter the race
        threads[first_leader].join()
        threads[first_leader] = self._spawn_contender(first_leader,
                                                      elections[first_leader])

        # contender set should now be the current leader plus the first leader
        wait(lambda: len(election.contenders()) == 2)
        contenders = election.contenders()
        eq_(set(contenders), set([self.leader_id, first_leader]))

        # make current leader raise an exception. first should be reelected
        self.raise_exception = True
        self.exit_event.set()
        with self.condition:
            while self.leader_id != first_leader:
                self.condition.wait(45)
        eq_(self.leader_id, first_leader)
        self._check_thread_error()

        self.exit_event.set()
        for thread in threads.values():
            thread.join()
        self._check_thread_error()
Exemple #15
0
    def test_election(self):
        elections = {}
        threads = {}
        for _ in range(3):
            contender = "c" + uuid.uuid4().hex
            elections[contender] = self.client.Election(self.path, contender)
            threads[contender] = self._spawn_contender(contender,
                elections[contender])

        # wait for a leader to be elected
        times = 0
        with self.condition:
            while not self.leader_id:
                self.condition.wait(5)
                times += 1
                if times > 5:
                    raise Exception("Still not a leader: lid: %s",
                                    self.leader_id)

        election = self.client.Election(self.path)

        # make sure all contenders are in the pool
        wait(lambda: len(election.contenders()) == len(elections))
        contenders = election.contenders()

        eq_(set(contenders), set(elections.keys()))

        # first one in list should be leader
        first_leader = contenders[0]
        eq_(first_leader, self.leader_id)

        # tell second one to cancel election. should never get elected.
        elections[contenders[1]].cancel()

        # make leader exit. third contender should be elected.
        self.exit_event.set()
        with self.condition:
            while self.leader_id == first_leader:
                self.condition.wait(45)
        eq_(self.leader_id, contenders[2])
        self._check_thread_error()

        # make first contender re-enter the race
        threads[first_leader].join()
        threads[first_leader] = self._spawn_contender(first_leader,
            elections[first_leader])

        # contender set should now be the current leader plus the first leader
        wait(lambda: len(election.contenders()) == 2)
        contenders = election.contenders()
        eq_(set(contenders), set([self.leader_id, first_leader]))

        # make current leader raise an exception. first should be reelected
        self.raise_exception = True
        self.exit_event.set()
        with self.condition:
            while self.leader_id != first_leader:
                self.condition.wait(45)
        eq_(self.leader_id, first_leader)
        self._check_thread_error()

        self.exit_event.set()
        for thread in threads.values():
            thread.join()
        self._check_thread_error()
Exemple #16
0
    def _do_test(self, contender_count, exclusive_mod=0,
                 revocation_mod=0, callback_mod=0,
                 callback_action_mod=0):
        """Runs a shared lock test based on the parameters given. Allows
        testing different combinations of shared vs exclusive locks, locks that
        request revocations, and locks that receive callbacks. This is useful
        given the range of lock strategies offered by ``Lock``.

        The method computes the expected groups of locks that should occur
        given the parameters. It then tests the state of the lock system at
        each of these phases, such as which locks have been acquired, which
        are pending, whether revocations have been issued, and whether a single
        revocation callback has been received for impacted locks.

        The main parameter is ``contender_count``, which indicates how many
        concurrent lock contenders with associated threads will be created. The
        remaining parameters indicate the modulus interval (of the contender
        identifier) that will attract a particular behavior. For example, a
        ``contender_count`` of 20 with an ``exclusive_mod`` of 6 will produce
        lock contenders numbered 0 to 19, and contenders numbered 0, 6, 12 and
        18 will request exclusive locks. Use modulus 0 to disable a behaviour.

        :param contender_count: number of contenders to create
        :param exclusive_mod: the modulus that will be used to produce
                              exclusive locks (use 1 to make all contenders use
                              exclusive locks, or 0 for all contenders to use
                              shared locks)
        :param revocation_mod: the modulus that will request revocation of any
                               existing lock
        :param callback_mod: the modulus that will request an unlock callback

        """
        # record the lock contenders we're creating
        def build_contenders(contender_count, modulus):
            if modulus == 0:
                return []
            return ["contender" + str(i) for i in range(contender_count)
                    if i % modulus == 0]

        names = build_contenders(contender_count, 1)
        exclusives = build_contenders(contender_count, exclusive_mod)
        revokers = build_contenders(contender_count, revocation_mod)
        callbacks = build_contenders(contender_count, callback_mod)

        # compute the phases (ie groups of concurrent locks) that should occur
        phases = []
        this_phase = []
        for name in names:
            if name in exclusives:
                if len(this_phase) > 0:
                    phases.append(this_phase)  # store last phase
                phases.append([name])  # record a phase with just the exclusive
                this_phase = []  # start new phase
            else:
                this_phase.append(name)  # shared
        phases.append(this_phase)  # store the final phase

        # setup our contenders
        threads = []
        contender_bits = {}
        for name in names:
            e = threading.Event()

            exclusive = True if name in exclusives else False
            l = self.client.Lock(self.lockpath, name, exclusive=exclusive)

            revoke = True if name in revokers else False
            callback = True if name in callbacks else False
            revoke = True if name in revokers else False
            t = threading.Thread(target=self._thread_lock_acquire_til_event,
                                 args=(name, l, e, revoke, callback))
            contender_bits[name] = (t, e)
            threads.append(t)

        # acquire the lock ourselves first to make the others line up
        lock = self.client.Lock(self.lockpath, "test")
        lock.acquire()

        # start threads one-by-one to prevent thread scheduling inconsistencies
        for contender in names:
            thread, event = contender_bits[contender]
            thread.start()
            wait(lambda: contender in lock.contenders())

        contenders = lock.contenders()
        eq_(contenders[0], "test")
        contenders = contenders[1:]
        remaining = list(contenders)

        # release the lock and contenders should claim it in order
        lock.release()

        # validate state at each phase
        for phase in phases:
            for contender in phase:
                thread, event = contender_bits[contender]

                with self.condition:
                    while not contender in self.active_threads:
                        self.condition.wait()

            # all claim to have their locks, so check expected remainders
            eq_(lock.contenders(), remaining)
            remaining = [x for x in remaining if x not in phase]

            unlocks = lock.contenders(unlocks_only=True)
            if not set(remaining).isdisjoint(revokers):
                # 1+ revoker is remaining, so revocations should have been made
                # for at least all members of this phase
                ok_(set(phase).issubset(set(unlocks)))

                # in addition all members of this phase with callbacks should
                # have fired
                for expected_callback in set(phase).intersection(callbacks):
                    with self.condition:
                        while not expected_callback in self.callback_threads:
                            self.condition.wait()

            # release their locks
            for contender in phase:
                thread, event = contender_bits[contender]
                event.set()

                with self.condition:
                    while contender in self.active_threads:
                        self.condition.wait()

        # ensure all terminate
        for thread in threads:
            thread.join()