コード例 #1
0
    def test_greenlet_sockets(self):
        # Check that Pool gives two sockets to two greenlets
        try:
            import greenlet
        except ImportError:
            raise SkipTest('greenlet not installed')

        cx_pool = pool.Pool(pair=(host, port),
                            max_size=10,
                            net_timeout=1000,
                            conn_timeout=1000,
                            use_ssl=False,
                            use_greenlets=True)

        socks = []

        def get_socket():
            cx_pool.start_request()
            socks.append(cx_pool.get_socket())

        looplet([
            greenlet.greenlet(get_socket),
            greenlet.greenlet(get_socket),
        ])

        self.assertEqual(2, len(socks))
        self.assertNotEqual(socks[0], socks[1])
コード例 #2
0
    def test_greenlet_sockets(self):
        # Check that Pool gives two sockets to two greenlets
        try:
            import greenlet
            import gevent
        except ImportError:
            raise SkipTest('gevent not installed')

        from pymongo import thread_util_gevent

        cx_pool = pool.Pool(
            pair=(host,port),
            max_size=10,
            net_timeout=1000,
            conn_timeout=1000,
            use_ssl=False,
            thread_support_module=thread_util_gevent)

        socks = []

        def get_socket():
            cx_pool.start_request()
            socks.append(cx_pool.get_socket())

        looplet([
            greenlet.greenlet(get_socket),
            greenlet.greenlet(get_socket),
        ])

        self.assertEqual(2, len(socks))
        self.assertNotEqual(socks[0], socks[1])
コード例 #3
0
    def _test_counter(self, use_greenlets):
        if use_greenlets:
            from pymongo import thread_util_gevent
            thread_support_module = thread_util_gevent
        else:
            from pymongo import thread_util_threading
            thread_support_module = thread_util_threading
        counter = thread_util.Counter(thread_support_module)

        self.assertEqual(0, counter.dec())
        self.assertEqual(0, counter.get())
        self.assertEqual(0, counter.dec())
        self.assertEqual(0, counter.get())

        done = set()

        def f(n):
            for i in xrange(n):
                self.assertEqual(i, counter.get())
                self.assertEqual(i + 1, counter.inc())

            for i in xrange(n, 0, -1):
                self.assertEqual(i, counter.get())
                self.assertEqual(i - 1, counter.dec())

            self.assertEqual(0, counter.get())

            # Extra decrements have no effect
            self.assertEqual(0, counter.dec())
            self.assertEqual(0, counter.get())
            self.assertEqual(0, counter.dec())
            self.assertEqual(0, counter.get())

            done.add(n)

        if use_greenlets:
            greenlets = [
                greenlet.greenlet(my_partial(f, i)) for i in xrange(10)]
            looplet(greenlets)
        else:
            threads = [
                threading.Thread(target=my_partial(f, i)) for i in xrange(10)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()

        self.assertEqual(10, len(done))
コード例 #4
0
    def _test_counter(self, use_greenlets):
        counter = thread_util.Counter(use_greenlets)

        self.assertEqual(0, counter.dec())
        self.assertEqual(0, counter.get())
        self.assertEqual(0, counter.dec())
        self.assertEqual(0, counter.get())

        done = set()

        def f(n):
            for i in xrange(n):
                self.assertEqual(i, counter.get())
                self.assertEqual(i + 1, counter.inc())

            for i in xrange(n, 0, -1):
                self.assertEqual(i, counter.get())
                self.assertEqual(i - 1, counter.dec())

            self.assertEqual(0, counter.get())

            # Extra decrements have no effect
            self.assertEqual(0, counter.dec())
            self.assertEqual(0, counter.get())
            self.assertEqual(0, counter.dec())
            self.assertEqual(0, counter.get())

            done.add(n)

        if use_greenlets:
            greenlets = [
                greenlet.greenlet(my_partial(f, i)) for i in xrange(10)
            ]
            looplet(greenlets)
        else:
            threads = [
                threading.Thread(target=my_partial(f, i)) for i in xrange(10)
            ]
            for t in threads:
                t.start()
            for t in threads:
                t.join()

        self.assertEqual(10, len(done))
コード例 #5
0
    def _test_counter(self, use_greenlets):
        counter = thread_util.Counter(use_greenlets)

        self.assertEqual(0, counter.dec())
        self.assertEqual(0, counter.get())
        self.assertEqual(0, counter.dec())
        self.assertEqual(0, counter.get())

        done = set()

        def f(n):
            for i in xrange(n):
                self.assertEqual(i, counter.get())
                self.assertEqual(i + 1, counter.inc())

            for i in xrange(n, 0, -1):
                self.assertEqual(i, counter.get())
                self.assertEqual(i - 1, counter.dec())

            self.assertEqual(0, counter.get())

            # Extra decrements have no effect
            self.assertEqual(0, counter.dec())
            self.assertEqual(0, counter.get())
            self.assertEqual(0, counter.dec())
            self.assertEqual(0, counter.get())

            done.add(n)

        if use_greenlets:
            greenlets = [
                greenlet.greenlet(partial(f, i)) for i in xrange(10)]
            looplet(greenlets)
        else:
            threads = [
                threading.Thread(target=partial(f, i)) for i in xrange(10)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()

        self.assertEqual(10, len(done))
コード例 #6
0
    def test_greenlet_sockets(self):
        # Check that Pool gives two sockets to two greenlets
        try:
            import greenlet
        except ImportError:
            raise SkipTest("greenlet not installed")

        cx_pool = pool.Pool(
            pair=(host, port), max_size=10, net_timeout=1000, conn_timeout=1000, use_ssl=False, use_greenlets=True
        )

        socks = []

        def get_socket():
            cx_pool.start_request()
            socks.append(cx_pool.get_socket())

        looplet([greenlet.greenlet(get_socket), greenlet.greenlet(get_socket)])

        self.assertEqual(2, len(socks))
        self.assertNotEqual(socks[0], socks[1])
コード例 #7
0
ファイル: test_thread_util.py プロジェクト: AlexSnet/oneline
    def _test_ident(self, use_greenlets):
        if 'java' in sys.platform:
            raise SkipTest("Can't rely on weakref callbacks in Jython")

        ident = thread_util.create_ident(use_greenlets)

        ids = set([ident.get()])
        unwatched_id = []
        done = set([ident.get()])  # Start with main thread's / greenlet's id.
        died = set()

        class Watched(object):
            def __init__(self, ident):
                self._my_ident = ident

            def before_rendezvous(self):
                self.my_id = self._my_ident.get()
                ids.add(self.my_id)

            def after_rendezvous(self):
                assert not self._my_ident.watching()
                self._my_ident.watch(lambda ref: died.add(self.my_id))
                assert self._my_ident.watching()
                done.add(self.my_id)

        class Unwatched(Watched):
            def before_rendezvous(self):
                Watched.before_rendezvous(self)
                unwatched_id.append(self.my_id)

            def after_rendezvous(self):
                Watched.after_rendezvous(self)
                self._my_ident.unwatch(self.my_id)
                assert not self._my_ident.watching()

        if use_greenlets:
            class WatchedGreenlet(Watched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            class UnwatchedGreenlet(Unwatched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
            t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
            looplet([t_watched, t_unwatched])
        else:
            class WatchedThread(Watched, RendezvousThread):
                def __init__(self, ident, state):
                    Watched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            class UnwatchedThread(Unwatched, RendezvousThread):
                def __init__(self, ident, state):
                    Unwatched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            state = RendezvousThread.create_shared_state(2)
            t_watched = WatchedThread(ident, state)
            t_watched.start()

            t_unwatched = UnwatchedThread(ident, state)
            t_unwatched.start()

            RendezvousThread.wait_for_rendezvous(state)
            RendezvousThread.resume_after_rendezvous(state)

            t_watched.join()
            t_unwatched.join()

            self.assertTrue(t_watched.passed)
            self.assertTrue(t_unwatched.passed)

        # Remove references, let weakref callbacks run
        del t_watched
        del t_unwatched

        # Trigger final cleanup in Python <= 2.7.0.
        # http://bugs.python.org/issue1868
        ident.get()
        self.assertEqual(3, len(ids))
        self.assertEqual(3, len(done))

        # Make sure thread is really gone
        slept = 0
        while not died and slept < 10:
            time.sleep(1)
            gc.collect()
            slept += 1

        self.assertEqual(1, len(died))
        self.assertFalse(unwatched_id[0] in died)
コード例 #8
0
    def test_greenlet_sockets_with_request(self):
        # Verify two assumptions: that start_request() with two greenlets but
        # not use_greenlets fails, meaning that the two greenlets will
        # share one socket. Also check that start_request() with use_greenlets
        # succeeds, meaning that two greenlets will get different sockets.

        try:
            import greenlet
            import gevent
        except ImportError:
            raise SkipTest('gevent not installed')

        from pymongo import thread_util_threading, thread_util_gevent

        pool_args = dict(
            pair=(host,port),
            max_size=10,
            net_timeout=1000,
            conn_timeout=1000,
            use_ssl=False,
        )

        for use_greenlets, use_request, expect_success in [
            (True, True, True),
            (True, False, False),
            (False, True, False),
            (False, False, False),
        ]:
            pool_args_cp = pool_args.copy()
            if use_greenlets:
                pool_args_cp['thread_support_module'] = thread_util_gevent
            else:
                pool_args_cp['thread_support_module'] = thread_util_threading

            cx_pool = pool.Pool(**pool_args_cp)

            # Map: greenlet -> socket
            greenlet2socks = {}
            main = greenlet.getcurrent()

            def get_socket_in_request():
                # Get a socket from the pool twice, switching contexts each time
                if use_request:
                    cx_pool.start_request()

                main.switch()

                for _ in range(2):
                    sock = cx_pool.get_socket()
                    cx_pool.maybe_return_socket(sock)
                    greenlet2socks.setdefault(
                        greenlet.getcurrent(), []
                    ).append(id(sock))

                    main.switch()

                cx_pool.end_request()

            greenlets = [
                greenlet.greenlet(get_socket_in_request),
                greenlet.greenlet(get_socket_in_request),
            ]

            # Run both greenlets to completion
            looplet(greenlets)

            socks_for_gr0 = greenlet2socks[greenlets[0]]
            socks_for_gr1 = greenlet2socks[greenlets[1]]

            # Whether we expect requests to work or not, we definitely expect
            # greenlet2socks to have the same number of keys and values
            self.assertEqual(2, len(greenlet2socks))
            self.assertEqual(2, len(socks_for_gr0))
            self.assertEqual(2, len(socks_for_gr1))

            # If we started a request, then there was a point at which we had
            # 2 active sockets, otherwise we always used one.
            if use_request and use_greenlets:
                self.assertEqual(2, len(cx_pool.sockets))
            else:
                self.assertEqual(1, len(cx_pool.sockets))

            # Again, regardless of whether requests work, a greenlet will get
            # the same socket each time it calls get_socket() within a request.
            # What we're really testing is that the two *different* greenlets
            # get *different* sockets from each other.
            self.assertEqual(
                socks_for_gr0[0], socks_for_gr0[1],
                "Expected greenlet 0 to get the same socket for each call "
                "to get_socket()"
            )

            self.assertEqual(
                socks_for_gr1[0], socks_for_gr1[1],
                "Expected greenlet 1 to get the same socket for each call "
                "to get_socket()"
            )

            if expect_success:
                # We passed use_greenlets=True, so start_request successfully
                # distinguished between the two greenlets.
                self.assertNotEqual(
                    socks_for_gr0[0], socks_for_gr1[0],
                    "Expected two greenlets to get two different sockets"
                )

            else:
                # We passed use_greenlets=False, so start_request didn't
                # distinguish between the two greenlets, and it gave them both
                # the same socket.
                self.assertEqual(
                    socks_for_gr0[0], socks_for_gr1[0],
                    "Expected two greenlets to get same socket"
                )
コード例 #9
0
    def _test_ident(self, use_greenlets):
        ident = thread_util.create_ident(use_greenlets)

        ids = set([ident.get()])
        unwatched_id = []
        done = set([ident.get()])  # Start with main thread's / greenlet's id.
        died = set()

        class Watched(object):
            def __init__(self, ident):
                self._my_ident = ident

            def before_rendezvous(self):
                self.my_id = self._my_ident.get()
                ids.add(self.my_id)

            def after_rendezvous(self):
                assert not self._my_ident.watching()
                self._my_ident.watch(lambda ref: died.add(self.my_id))
                assert self._my_ident.watching()
                done.add(self.my_id)

        class Unwatched(Watched):
            def before_rendezvous(self):
                Watched.before_rendezvous(self)
                unwatched_id.append(self.my_id)

            def after_rendezvous(self):
                Watched.after_rendezvous(self)
                self._my_ident.unwatch(self.my_id)
                assert not self._my_ident.watching()

        if use_greenlets:

            class WatchedGreenlet(Watched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            class UnwatchedGreenlet(Unwatched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
            t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
            looplet([t_watched, t_unwatched])
        else:

            class WatchedThread(Watched, RendezvousThread):
                def __init__(self, ident, state):
                    Watched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            class UnwatchedThread(Unwatched, RendezvousThread):
                def __init__(self, ident, state):
                    Unwatched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            state = RendezvousThread.create_shared_state(2)
            t_watched = WatchedThread(ident, state)
            t_watched.start()

            t_unwatched = UnwatchedThread(ident, state)
            t_unwatched.start()

            RendezvousThread.wait_for_rendezvous(state)
            RendezvousThread.resume_after_rendezvous(state)

            t_watched.join()
            t_unwatched.join()

            self.assertTrue(t_watched.passed)
            self.assertTrue(t_unwatched.passed)

        # Remove references, let weakref callbacks run
        del t_watched
        del t_unwatched

        # Trigger final cleanup in Python <= 2.7.0.
        # http://bugs.python.org/issue1868
        ident.get()
        self.assertEqual(3, len(ids))
        self.assertEqual(3, len(done))

        # Make sure thread is really gone
        slept = 0
        while not died and slept < 10:
            time.sleep(1)
            gc.collect()
            slept += 1

        self.assertEqual(1, len(died))
        self.assertFalse(unwatched_id[0] in died)
コード例 #10
0
    def test_greenlet_sockets_with_request(self):
        # Verify two assumptions: that start_request() with two greenlets but
        # not use_greenlets fails, meaning that the two greenlets will
        # share one socket. Also check that start_request() with use_greenlets
        # succeeds, meaning that two greenlets will get different sockets.

        try:
            import greenlet
        except ImportError:
            raise SkipTest('greenlet not installed')

        pool_args = dict(
            pair=(host, port),
            max_size=10,
            net_timeout=1000,
            conn_timeout=1000,
            use_ssl=False,
        )

        for use_greenlets, use_request, expect_success in [
            (True, True, True),
            (True, False, False),
            (False, True, False),
            (False, False, False),
        ]:
            pool_args_cp = pool_args.copy()
            pool_args_cp['use_greenlets'] = use_greenlets
            cx_pool = pool.Pool(**pool_args_cp)

            # Map: greenlet -> socket
            greenlet2socks = {}
            main = greenlet.getcurrent()

            def get_socket_in_request():
                # Get a socket from the pool twice, switching contexts each time
                if use_request:
                    cx_pool.start_request()

                main.switch()

                for _ in range(2):
                    sock = cx_pool.get_socket()
                    cx_pool.maybe_return_socket(sock)
                    greenlet2socks.setdefault(greenlet.getcurrent(),
                                              []).append(id(sock))

                    main.switch()

                cx_pool.end_request()

            greenlets = [
                greenlet.greenlet(get_socket_in_request),
                greenlet.greenlet(get_socket_in_request),
            ]

            # Run both greenlets to completion
            looplet(greenlets)

            socks_for_gr0 = greenlet2socks[greenlets[0]]
            socks_for_gr1 = greenlet2socks[greenlets[1]]

            # Whether we expect requests to work or not, we definitely expect
            # greenlet2socks to have the same number of keys and values
            self.assertEqual(2, len(greenlet2socks))
            self.assertEqual(2, len(socks_for_gr0))
            self.assertEqual(2, len(socks_for_gr1))

            # If we started a request, then there was a point at which we had
            # 2 active sockets, otherwise we always used one.
            if use_request and use_greenlets:
                self.assertEqual(2, len(cx_pool.sockets))
            else:
                self.assertEqual(1, len(cx_pool.sockets))

            # Again, regardless of whether requests work, a greenlet will get
            # the same socket each time it calls get_socket() within a request.
            # What we're really testing is that the two *different* greenlets
            # get *different* sockets from each other.
            self.assertEqual(
                socks_for_gr0[0], socks_for_gr0[1],
                "Expected greenlet 0 to get the same socket for each call "
                "to get_socket()")

            self.assertEqual(
                socks_for_gr1[0], socks_for_gr1[1],
                "Expected greenlet 1 to get the same socket for each call "
                "to get_socket()")

            if expect_success:
                # We passed use_greenlets=True, so start_request successfully
                # distinguished between the two greenlets.
                self.assertNotEqual(
                    socks_for_gr0[0], socks_for_gr1[0],
                    "Expected two greenlets to get two different sockets")

            else:
                # We passed use_greenlets=False, so start_request didn't
                # distinguish between the two greenlets, and it gave them both
                # the same socket.
                self.assertEqual(socks_for_gr0[0], socks_for_gr1[0],
                                 "Expected two greenlets to get same socket")