示例#1
0
    def test_unwatch_cleans_up(self):
        # GreenletIdent.unwatch() should remove the on_thread_died callback
        # from an enhanced Gevent Greenlet's list of links.
        callback_ran = [False]

        def on_greenlet_died(_):
            callback_ran[0] = True

        ident = thread_util.create_ident(use_greenlets=True)

        def watch_and_unwatch():
            ident.watch(on_greenlet_died)
            ident.unwatch(ident.get())

        g = gevent.greenlet.Greenlet(run=watch_and_unwatch)
        g.start()
        g.join(10)
        the_hub = gevent.hub.get_hub()
        if hasattr(the_hub, 'join'):
            # Gevent 1.0
            the_hub.join()
        else:
            # Gevent 0.13 and less
            the_hub.shutdown()

        self.assertTrue(g.successful())

        # unwatch() canceled the callback.
        self.assertFalse(callback_ran[0])
示例#2
0
    def test_unwatch_cleans_up(self):
        # GreenletIdent.unwatch() should remove the on_thread_died callback
        # from an enhanced Gevent Greenlet's list of links.
        callback_ran = [False]

        def on_greenlet_died(_):
            callback_ran[0] = True

        ident = thread_util.create_ident(use_greenlets=True)

        def watch_and_unwatch():
            ident.watch(on_greenlet_died)
            ident.unwatch(ident.get())

        g = gevent.greenlet.Greenlet(run=watch_and_unwatch)
        g.start()
        g.join(10)
        the_hub = gevent.hub.get_hub()
        if hasattr(the_hub, 'join'):
            # Gevent 1.0
            the_hub.join()
        else:
            # Gevent 0.13 and less
            the_hub.shutdown()

        self.assertTrue(g.successful())

        # unwatch() canceled the callback.
        self.assertFalse(callback_ran[0])
示例#3
0
    def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl, use_greenlets):
        """
        :Parameters:
          - `pair`: a (hostname, port) tuple
          - `max_size`: approximate number of idle connections to keep open
          - `net_timeout`: timeout in seconds for operations on open connection
          - `conn_timeout`: timeout in seconds for establishing connection
          - `use_ssl`: bool, if True use an encrypted connection
          - `use_greenlets`: bool, if True then start_request() assigns a
              socket to the current greenlet - otherwise it is assigned to the
              current thread
        """
        if use_greenlets and not thread_util.have_greenlet:
            raise ConfigurationError("The greenlet module is not available. " "Install the greenlet package from PyPI.")

        self.sockets = set()
        self.lock = threading.Lock()

        # Keep track of resets, so we notice sockets created before the most
        # recent reset and close them.
        self.pool_id = 0
        self.pid = os.getpid()
        self.pair = pair
        self.max_size = max_size
        self.net_timeout = net_timeout
        self.conn_timeout = conn_timeout
        self.use_ssl = use_ssl
        self._ident = thread_util.create_ident(use_greenlets)

        # Map self._ident.get() -> request socket
        self._tid_to_sock = {}

        # Count the number of calls to start_request() per thread or greenlet
        self._request_counter = thread_util.Counter(use_greenlets)
示例#4
0
    def _test_ident(self, use_greenlets):
        if 'java' in sys.platform:
            raise SkipTest("Can't rely on weakref callbacks in Jython")

        ident = thread_util.create_ident(use_greenlets)

        ids = set([ident.get()])
        unwatched_id = []
        done = set([ident.get()])  # Start with main thread's / greenlet's id.
        died = set()

        class Watched(object):
            def __init__(self, ident):
                self._my_ident = ident

            def before_rendezvous(self):
                self.my_id = self._my_ident.get()
                ids.add(self.my_id)

            def after_rendezvous(self):
                assert not self._my_ident.watching()
                self._my_ident.watch(lambda ref: died.add(self.my_id))
                assert self._my_ident.watching()
                done.add(self.my_id)

        class Unwatched(Watched):
            def before_rendezvous(self):
                Watched.before_rendezvous(self)
                unwatched_id.append(self.my_id)

            def after_rendezvous(self):
                Watched.after_rendezvous(self)
                self._my_ident.unwatch(self.my_id)
                assert not self._my_ident.watching()

        if use_greenlets:
            class WatchedGreenlet(Watched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            class UnwatchedGreenlet(Unwatched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
            t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
            looplet([t_watched, t_unwatched])
        else:
            class WatchedThread(Watched, RendezvousThread):
                def __init__(self, ident, state):
                    Watched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            class UnwatchedThread(Unwatched, RendezvousThread):
                def __init__(self, ident, state):
                    Unwatched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            state = RendezvousThread.create_shared_state(2)
            t_watched = WatchedThread(ident, state)
            t_watched.start()

            t_unwatched = UnwatchedThread(ident, state)
            t_unwatched.start()

            RendezvousThread.wait_for_rendezvous(state)
            RendezvousThread.resume_after_rendezvous(state)

            t_watched.join()
            t_unwatched.join()

            self.assertTrue(t_watched.passed)
            self.assertTrue(t_unwatched.passed)

        # Remove references, let weakref callbacks run
        del t_watched
        del t_unwatched

        # Trigger final cleanup in Python <= 2.7.0.
        # http://bugs.python.org/issue1868
        ident.get()
        self.assertEqual(3, len(ids))
        self.assertEqual(3, len(done))

        # Make sure thread is really gone
        slept = 0
        while not died and slept < 10:
            time.sleep(1)
            gc.collect()
            slept += 1

        self.assertEqual(1, len(died))
        self.assertFalse(unwatched_id[0] in died)
示例#5
0
    def __init__(self,
                 pair,
                 max_size,
                 net_timeout,
                 conn_timeout,
                 use_ssl,
                 use_greenlets,
                 ssl_keyfile=None,
                 ssl_certfile=None,
                 ssl_cert_reqs=None,
                 ssl_ca_certs=None,
                 wait_queue_timeout=None,
                 wait_queue_multiple=None):
        """
        :Parameters:
          - `pair`: a (hostname, port) tuple
          - `max_size`: The maximum number of open sockets. Calls to
            `get_socket` will block if this is set, this pool has opened
            `max_size` sockets, and there are none idle. Set to `None` to
             disable.
          - `net_timeout`: timeout in seconds for operations on open connection
          - `conn_timeout`: timeout in seconds for establishing connection
          - `use_ssl`: bool, if True use an encrypted connection
          - `use_greenlets`: bool, if True then start_request() assigns a
              socket to the current greenlet - otherwise it is assigned to the
              current thread
          - `ssl_keyfile`: The private keyfile used to identify the local
            connection against mongod.  If included with the ``certfile` then
            only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
          - `ssl_certfile`: The certificate file used to identify the local
            connection against mongod. Implies ``ssl=True``.
          - `ssl_cert_reqs`: Specifies whether a certificate is required from
            the other side of the connection, and whether it will be validated
            if provided. It must be one of the three values ``ssl.CERT_NONE``
            (certificates ignored), ``ssl.CERT_OPTIONAL``
            (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
            (required and validated). If the value of this parameter is not
            ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
            to a file of CA certificates. Implies ``ssl=True``.
          - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
            "certification authority" certificates, which are used to validate
            certificates passed from the other end of the connection.
            Implies ``ssl=True``.
          - `wait_queue_timeout`: (integer) How long (in seconds) a
            thread will wait for a socket from the pool if the pool has no
            free sockets.
          - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
            the number of threads allowed to wait for a socket at one time.
        """
        # Only check a socket's health with _closed() every once in a while.
        # Can override for testing: 0 to always check, None to never check.
        self._check_interval_seconds = 1

        self.sockets = set()
        self.lock = threading.Lock()

        # Keep track of resets, so we notice sockets created before the most
        # recent reset and close them.
        self.pool_id = 0
        self.pid = os.getpid()
        self.pair = pair
        self.max_size = max_size
        self.net_timeout = net_timeout
        self.conn_timeout = conn_timeout
        self.wait_queue_timeout = wait_queue_timeout
        self.wait_queue_multiple = wait_queue_multiple
        self.use_ssl = use_ssl
        self.ssl_keyfile = ssl_keyfile
        self.ssl_certfile = ssl_certfile
        self.ssl_cert_reqs = ssl_cert_reqs
        self.ssl_ca_certs = ssl_ca_certs

        if HAS_SSL and use_ssl and not ssl_cert_reqs:
            self.ssl_cert_reqs = ssl.CERT_NONE

        # Map self._ident.get() -> request socket
        self._tid_to_sock = {}

        if use_greenlets and not thread_util.have_gevent:
            raise ConfigurationError("The Gevent module is not available. "
                                     "Install the gevent package from PyPI.")

        self._ident = thread_util.create_ident(use_greenlets)

        # Count the number of calls to start_request() per thread or greenlet
        self._request_counter = thread_util.Counter(use_greenlets)

        if self.wait_queue_multiple is None or self.max_size is None:
            max_waiters = None
        else:
            max_waiters = self.max_size * self.wait_queue_multiple

        self._socket_semaphore = thread_util.create_semaphore(
            self.max_size, max_waiters, use_greenlets)
示例#6
0
    def __init__(self,
                 pair,
                 max_size,
                 net_timeout,
                 conn_timeout,
                 use_ssl,
                 use_greenlets,
                 ssl_keyfile=None,
                 ssl_certfile=None,
                 ssl_cert_reqs=None,
                 ssl_ca_certs=None,
                 wait_queue_timeout=None,
                 wait_queue_multiple=None,
                 socket_keepalive=False,
                 ssl_match_hostname=True):
        """
        :Parameters:
          - `pair`: a (hostname, port) tuple
          - `max_size`: The maximum number of open sockets. Calls to
            `get_socket` will block if this is set, this pool has opened
            `max_size` sockets, and there are none idle. Set to `None` to
             disable.
          - `net_timeout`: timeout in seconds for operations on open connection
          - `conn_timeout`: timeout in seconds for establishing connection
          - `use_ssl`: bool, if True use an encrypted connection
          - `use_greenlets`: bool, if True then start_request() assigns a
              socket to the current greenlet - otherwise it is assigned to the
              current thread
          - `ssl_keyfile`: The private keyfile used to identify the local
            connection against mongod.  If included with the ``certfile` then
            only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
          - `ssl_certfile`: The certificate file used to identify the local
            connection against mongod. Implies ``ssl=True``.
          - `ssl_cert_reqs`: Specifies whether a certificate is required from
            the other side of the connection, and whether it will be validated
            if provided. It must be one of the three values ``ssl.CERT_NONE``
            (certificates ignored), ``ssl.CERT_OPTIONAL``
            (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
            (required and validated). If the value of this parameter is not
            ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
            to a file of CA certificates. Implies ``ssl=True``.
          - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
            "certification authority" certificates, which are used to validate
            certificates passed from the other end of the connection.
            Implies ``ssl=True``.
          - `wait_queue_timeout`: (integer) How long (in seconds) a
            thread will wait for a socket from the pool if the pool has no
            free sockets.
          - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
            the number of threads allowed to wait for a socket at one time.
          - `socket_keepalive`: (boolean) Whether to send periodic keep-alive
            packets on connected sockets. Defaults to ``False`` (do not send
            keep-alive packets).
          - `ssl_match_hostname`: If ``True`` (the default), and
            `ssl_cert_reqs` is not ``ssl.CERT_NONE``, enables hostname
            verification using the :func:`~ssl.match_hostname` function from
            python's :mod:`~ssl` module. Think very carefully before setting
            this to ``False`` as that could make your application vulnerable to
            man-in-the-middle attacks.
        """
        # Only check a socket's health with _closed() every once in a while.
        # Can override for testing: 0 to always check, None to never check.
        self._check_interval_seconds = 1

        self.sockets = set()
        self.lock = threading.Lock()

        # Keep track of resets, so we notice sockets created before the most
        # recent reset and close them.
        self.pool_id = 0
        self.pid = os.getpid()
        self.pair = pair
        self.max_size = max_size
        self.net_timeout = net_timeout
        self.conn_timeout = conn_timeout
        self.wait_queue_timeout = wait_queue_timeout
        self.wait_queue_multiple = wait_queue_multiple
        self.socket_keepalive = socket_keepalive
        self.ssl_ctx = None
        self.ssl_match_hostname = ssl_match_hostname

        if HAS_SSL and use_ssl:
            # PROTOCOL_TLS_CLIENT added and PROTOCOL_SSLv23
            # deprecated in CPython 3.6
            self.ssl_ctx = _SSLContext(
                getattr(ssl, 'PROTOCOL_TLS_CLIENT', ssl.PROTOCOL_SSLv23))
            # SSLContext.check_hostname was added in 2.7.9 and 3.4. Using it
            # forces the use of SNI, which PyMongo 2.x doesn't support.
            # PROTOCOL_TLS_CLIENT enables this by default. Since we call
            # match_hostname directly disable this explicitly.
            if hasattr(self.ssl_ctx, "check_hostname"):
                self.ssl_ctx.check_hostname = False
            # load_cert_chain and load_verify_locations can fail
            # if the file contents are invalid.
            if ssl_certfile is not None:
                try:
                    self.ssl_ctx.load_cert_chain(ssl_certfile, ssl_keyfile)
                except ssl.SSLError:
                    pass
            if ssl_ca_certs is not None:
                try:
                    self.ssl_ctx.load_verify_locations(ssl_ca_certs)
                except ssl.SSLError:
                    pass
            # PROTOCOL_TLS_CLIENT sets verify_mode to CERT_REQUIRED so
            # we always have to set this explicitly.
            if ssl_cert_reqs is not None:
                self.ssl_ctx.verify_mode = ssl_cert_reqs
            else:
                self.ssl_ctx.verify_mode = ssl.CERT_NONE

        # Map self._ident.get() -> request socket
        self._tid_to_sock = {}

        if use_greenlets and not thread_util.have_gevent:
            raise ConfigurationError("The Gevent module is not available. "
                                     "Install the gevent package from PyPI.")

        self._ident = thread_util.create_ident(use_greenlets)

        # Count the number of calls to start_request() per thread or greenlet
        self._request_counter = thread_util.Counter(use_greenlets)

        if self.wait_queue_multiple is None or self.max_size is None:
            max_waiters = None
        else:
            max_waiters = self.max_size * self.wait_queue_multiple

        self._socket_semaphore = thread_util.create_semaphore(
            self.max_size, max_waiters, use_greenlets)
示例#7
0
    def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl,
                 use_greenlets, ssl_keyfile=None, ssl_certfile=None,
                 ssl_cert_reqs=None, ssl_ca_certs=None,
                 wait_queue_timeout=None, wait_queue_multiple=None):
        """
        :Parameters:
          - `pair`: a (hostname, port) tuple
          - `max_size`: The maximum number of open sockets. Calls to
            `get_socket` will block if this is set, this pool has opened
            `max_size` sockets, and there are none idle. Set to `None` to
             disable.
          - `net_timeout`: timeout in seconds for operations on open connection
          - `conn_timeout`: timeout in seconds for establishing connection
          - `use_ssl`: bool, if True use an encrypted connection
          - `use_greenlets`: bool, if True then start_request() assigns a
              socket to the current greenlet - otherwise it is assigned to the
              current thread
          - `ssl_keyfile`: The private keyfile used to identify the local
            connection against mongod.  If included with the ``certfile` then
            only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
          - `ssl_certfile`: The certificate file used to identify the local
            connection against mongod. Implies ``ssl=True``.
          - `ssl_cert_reqs`: Specifies whether a certificate is required from
            the other side of the connection, and whether it will be validated
            if provided. It must be one of the three values ``ssl.CERT_NONE``
            (certificates ignored), ``ssl.CERT_OPTIONAL``
            (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
            (required and validated). If the value of this parameter is not
            ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
            to a file of CA certificates. Implies ``ssl=True``.
          - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
            "certification authority" certificates, which are used to validate
            certificates passed from the other end of the connection.
            Implies ``ssl=True``.
          - `wait_queue_timeout`: (integer) How long (in seconds) a
            thread will wait for a socket from the pool if the pool has no
            free sockets.
          - `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
            the number of threads allowed to wait for a socket at one time.
        """
        # Only check a socket's health with _closed() every once in a while.
        # Can override for testing: 0 to always check, None to never check.
        self._check_interval_seconds = 1

        self.sockets = set()
        self.lock = threading.Lock()

        # Keep track of resets, so we notice sockets created before the most
        # recent reset and close them.
        self.pool_id = 0
        self.pid = os.getpid()
        self.pair = pair
        self.max_size = max_size
        self.net_timeout = net_timeout
        self.conn_timeout = conn_timeout
        self.wait_queue_timeout = wait_queue_timeout
        self.wait_queue_multiple = wait_queue_multiple
        self.use_ssl = use_ssl
        self.ssl_keyfile = ssl_keyfile
        self.ssl_certfile = ssl_certfile
        self.ssl_cert_reqs = ssl_cert_reqs
        self.ssl_ca_certs = ssl_ca_certs

        if HAS_SSL and use_ssl and not ssl_cert_reqs:
            self.ssl_cert_reqs = ssl.CERT_NONE

        # Map self._ident.get() -> request socket
        self._tid_to_sock = {}

        if use_greenlets and not thread_util.have_gevent:
            raise ConfigurationError(
                "The Gevent module is not available. "
                "Install the gevent package from PyPI."
            )

        self._ident = thread_util.create_ident(use_greenlets)

        # Count the number of calls to start_request() per thread or greenlet
        self._request_counter = thread_util.Counter(use_greenlets)

        if self.wait_queue_multiple is None or self.max_size is None:
            max_waiters = None
        else:
            max_waiters = self.max_size * self.wait_queue_multiple

        self._socket_semaphore = thread_util.create_semaphore(
            self.max_size, max_waiters, use_greenlets)
示例#8
0
    def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl,
                 use_greenlets, ssl_keyfile=None, ssl_certfile=None,
                 ssl_cert_reqs=None, ssl_ca_certs=None):
        """
        :Parameters:
          - `pair`: a (hostname, port) tuple
          - `max_size`: approximate number of idle connections to keep open
          - `net_timeout`: timeout in seconds for operations on open connection
          - `conn_timeout`: timeout in seconds for establishing connection
          - `use_ssl`: bool, if True use an encrypted connection
          - `use_greenlets`: bool, if True then start_request() assigns a
              socket to the current greenlet - otherwise it is assigned to the
              current thread
          - `ssl_keyfile`: The private keyfile used to identify the local
            connection against mongod.  If included with the ``certfile` then
            only the ``ssl_certfile`` is needed.  Implies ``ssl=True``.
          - `ssl_certfile`: The certificate file used to identify the local
            connection against mongod. Implies ``ssl=True``.
          - `ssl_cert_reqs`: Specifies whether a certificate is required from
            the other side of the connection, and whether it will be validated
            if provided. It must be one of the three values ``ssl.CERT_NONE``
            (certificates ignored), ``ssl.CERT_OPTIONAL``
            (not required, but validated if provided), or ``ssl.CERT_REQUIRED``
            (required and validated). If the value of this parameter is not
            ``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
            to a file of CA certificates. Implies ``ssl=True``.
          - `ssl_ca_certs`: The ca_certs file contains a set of concatenated
            "certification authority" certificates, which are used to validate
            certificates passed from the other end of the connection.
            Implies ``ssl=True``.
        """
        if use_greenlets and not thread_util.have_greenlet:
            raise ConfigurationError(
                "The greenlet module is not available. "
                "Install the greenlet package from PyPI."
            )

        self.sockets = set()
        self.lock = threading.Lock()

        # Keep track of resets, so we notice sockets created before the most
        # recent reset and close them.
        self.pool_id = 0
        self.pid = os.getpid()
        self.pair = pair
        self.max_size = max_size
        self.net_timeout = net_timeout
        self.conn_timeout = conn_timeout
        self.use_ssl = use_ssl
        self.ssl_keyfile = ssl_keyfile
        self.ssl_certfile = ssl_certfile
        self.ssl_cert_reqs = ssl_cert_reqs
        self.ssl_ca_certs = ssl_ca_certs

        if HAS_SSL and use_ssl and not ssl_cert_reqs:
            self.ssl_cert_reqs = ssl.CERT_NONE

        self._ident = thread_util.create_ident(use_greenlets)

        # Map self._ident.get() -> request socket
        self._tid_to_sock = {}

        # Count the number of calls to start_request() per thread or greenlet
        self._request_counter = thread_util.Counter(use_greenlets)
示例#9
0
    def _test_ident(self, use_greenlets):
        ident = thread_util.create_ident(use_greenlets)

        ids = set([ident.get()])
        unwatched_id = []
        done = set([ident.get()])  # Start with main thread's / greenlet's id.
        died = set()

        class Watched(object):
            def __init__(self, ident):
                self._my_ident = ident

            def before_rendezvous(self):
                self.my_id = self._my_ident.get()
                ids.add(self.my_id)

            def after_rendezvous(self):
                assert not self._my_ident.watching()
                self._my_ident.watch(lambda ref: died.add(self.my_id))
                assert self._my_ident.watching()
                done.add(self.my_id)

        class Unwatched(Watched):
            def before_rendezvous(self):
                Watched.before_rendezvous(self)
                unwatched_id.append(self.my_id)

            def after_rendezvous(self):
                Watched.after_rendezvous(self)
                self._my_ident.unwatch(self.my_id)
                assert not self._my_ident.watching()

        if use_greenlets:

            class WatchedGreenlet(Watched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            class UnwatchedGreenlet(Unwatched):
                def run(self):
                    self.before_rendezvous()
                    self.after_rendezvous()

            t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
            t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
            looplet([t_watched, t_unwatched])
        else:

            class WatchedThread(Watched, RendezvousThread):
                def __init__(self, ident, state):
                    Watched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            class UnwatchedThread(Unwatched, RendezvousThread):
                def __init__(self, ident, state):
                    Unwatched.__init__(self, ident)
                    RendezvousThread.__init__(self, state)

            state = RendezvousThread.create_shared_state(2)
            t_watched = WatchedThread(ident, state)
            t_watched.start()

            t_unwatched = UnwatchedThread(ident, state)
            t_unwatched.start()

            RendezvousThread.wait_for_rendezvous(state)
            RendezvousThread.resume_after_rendezvous(state)

            t_watched.join()
            t_unwatched.join()

            self.assertTrue(t_watched.passed)
            self.assertTrue(t_unwatched.passed)

        # Remove references, let weakref callbacks run
        del t_watched
        del t_unwatched

        # Trigger final cleanup in Python <= 2.7.0.
        # http://bugs.python.org/issue1868
        ident.get()
        self.assertEqual(3, len(ids))
        self.assertEqual(3, len(done))

        # Make sure thread is really gone
        slept = 0
        while not died and slept < 10:
            time.sleep(1)
            gc.collect()
            slept += 1

        self.assertEqual(1, len(died))
        self.assertFalse(unwatched_id[0] in died)