def test_unwatch_cleans_up(self):
# GreenletIdent.unwatch() should remove the on_thread_died callback
# from an enhanced Gevent Greenlet's list of links.
callback_ran = [False]
def on_greenlet_died(_):
callback_ran[0] = True
ident = thread_util.create_ident(use_greenlets=True)
def watch_and_unwatch():
ident.watch(on_greenlet_died)
ident.unwatch(ident.get())
g = gevent.greenlet.Greenlet(run=watch_and_unwatch)
g.start()
g.join(10)
the_hub = gevent.hub.get_hub()
if hasattr(the_hub, 'join'):
# Gevent 1.0
the_hub.join()
else:
# Gevent 0.13 and less
the_hub.shutdown()
self.assertTrue(g.successful())
# unwatch() canceled the callback.
self.assertFalse(callback_ran[0])
def test_unwatch_cleans_up(self):
# GreenletIdent.unwatch() should remove the on_thread_died callback
# from an enhanced Gevent Greenlet's list of links.
callback_ran = [False]
def on_greenlet_died(_):
callback_ran[0] = True
ident = thread_util.create_ident(use_greenlets=True)
def watch_and_unwatch():
ident.watch(on_greenlet_died)
ident.unwatch(ident.get())
g = gevent.greenlet.Greenlet(run=watch_and_unwatch)
g.start()
g.join(10)
the_hub = gevent.hub.get_hub()
if hasattr(the_hub, 'join'):
# Gevent 1.0
the_hub.join()
else:
# Gevent 0.13 and less
the_hub.shutdown()
self.assertTrue(g.successful())
# unwatch() canceled the callback.
self.assertFalse(callback_ran[0])
def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl, use_greenlets):
"""
:Parameters:
- `pair`: a (hostname, port) tuple
- `max_size`: approximate number of idle connections to keep open
- `net_timeout`: timeout in seconds for operations on open connection
- `conn_timeout`: timeout in seconds for establishing connection
- `use_ssl`: bool, if True use an encrypted connection
- `use_greenlets`: bool, if True then start_request() assigns a
socket to the current greenlet - otherwise it is assigned to the
current thread
"""
if use_greenlets and not thread_util.have_greenlet:
raise ConfigurationError("The greenlet module is not available. " "Install the greenlet package from PyPI.")
self.sockets = set()
self.lock = threading.Lock()
# Keep track of resets, so we notice sockets created before the most
# recent reset and close them.
self.pool_id = 0
self.pid = os.getpid()
self.pair = pair
self.max_size = max_size
self.net_timeout = net_timeout
self.conn_timeout = conn_timeout
self.use_ssl = use_ssl
self._ident = thread_util.create_ident(use_greenlets)
# Map self._ident.get() -> request socket
self._tid_to_sock = {}
# Count the number of calls to start_request() per thread or greenlet
self._request_counter = thread_util.Counter(use_greenlets)
def _test_ident(self, use_greenlets):
if 'java' in sys.platform:
raise SkipTest("Can't rely on weakref callbacks in Jython")
ident = thread_util.create_ident(use_greenlets)
ids = set([ident.get()])
unwatched_id = []
done = set([ident.get()]) # Start with main thread's / greenlet's id.
died = set()
class Watched(object):
def __init__(self, ident):
self._my_ident = ident
def before_rendezvous(self):
self.my_id = self._my_ident.get()
ids.add(self.my_id)
def after_rendezvous(self):
assert not self._my_ident.watching()
self._my_ident.watch(lambda ref: died.add(self.my_id))
assert self._my_ident.watching()
done.add(self.my_id)
class Unwatched(Watched):
def before_rendezvous(self):
Watched.before_rendezvous(self)
unwatched_id.append(self.my_id)
def after_rendezvous(self):
Watched.after_rendezvous(self)
self._my_ident.unwatch(self.my_id)
assert not self._my_ident.watching()
if use_greenlets:
class WatchedGreenlet(Watched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
class UnwatchedGreenlet(Unwatched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
looplet([t_watched, t_unwatched])
else:
class WatchedThread(Watched, RendezvousThread):
def __init__(self, ident, state):
Watched.__init__(self, ident)
RendezvousThread.__init__(self, state)
class UnwatchedThread(Unwatched, RendezvousThread):
def __init__(self, ident, state):
Unwatched.__init__(self, ident)
RendezvousThread.__init__(self, state)
state = RendezvousThread.create_shared_state(2)
t_watched = WatchedThread(ident, state)
t_watched.start()
t_unwatched = UnwatchedThread(ident, state)
t_unwatched.start()
RendezvousThread.wait_for_rendezvous(state)
RendezvousThread.resume_after_rendezvous(state)
t_watched.join()
t_unwatched.join()
self.assertTrue(t_watched.passed)
self.assertTrue(t_unwatched.passed)
# Remove references, let weakref callbacks run
del t_watched
del t_unwatched
# Trigger final cleanup in Python <= 2.7.0.
# http://bugs.python.org/issue1868
ident.get()
self.assertEqual(3, len(ids))
self.assertEqual(3, len(done))
# Make sure thread is really gone
slept = 0
while not died and slept < 10:
time.sleep(1)
gc.collect()
slept += 1
self.assertEqual(1, len(died))
self.assertFalse(unwatched_id[0] in died)
def __init__(self,
pair,
max_size,
net_timeout,
conn_timeout,
use_ssl,
use_greenlets,
ssl_keyfile=None,
ssl_certfile=None,
ssl_cert_reqs=None,
ssl_ca_certs=None,
wait_queue_timeout=None,
wait_queue_multiple=None):
"""
:Parameters:
- `pair`: a (hostname, port) tuple
- `max_size`: The maximum number of open sockets. Calls to
`get_socket` will block if this is set, this pool has opened
`max_size` sockets, and there are none idle. Set to `None` to
disable.
- `net_timeout`: timeout in seconds for operations on open connection
- `conn_timeout`: timeout in seconds for establishing connection
- `use_ssl`: bool, if True use an encrypted connection
- `use_greenlets`: bool, if True then start_request() assigns a
socket to the current greenlet - otherwise it is assigned to the
current thread
- `ssl_keyfile`: The private keyfile used to identify the local
connection against mongod. If included with the ``certfile` then
only the ``ssl_certfile`` is needed. Implies ``ssl=True``.
- `ssl_certfile`: The certificate file used to identify the local
connection against mongod. Implies ``ssl=True``.
- `ssl_cert_reqs`: Specifies whether a certificate is required from
the other side of the connection, and whether it will be validated
if provided. It must be one of the three values ``ssl.CERT_NONE``
(certificates ignored), ``ssl.CERT_OPTIONAL``
(not required, but validated if provided), or ``ssl.CERT_REQUIRED``
(required and validated). If the value of this parameter is not
``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
to a file of CA certificates. Implies ``ssl=True``.
- `ssl_ca_certs`: The ca_certs file contains a set of concatenated
"certification authority" certificates, which are used to validate
certificates passed from the other end of the connection.
Implies ``ssl=True``.
- `wait_queue_timeout`: (integer) How long (in seconds) a
thread will wait for a socket from the pool if the pool has no
free sockets.
- `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
the number of threads allowed to wait for a socket at one time.
"""
# Only check a socket's health with _closed() every once in a while.
# Can override for testing: 0 to always check, None to never check.
self._check_interval_seconds = 1
self.sockets = set()
self.lock = threading.Lock()
# Keep track of resets, so we notice sockets created before the most
# recent reset and close them.
self.pool_id = 0
self.pid = os.getpid()
self.pair = pair
self.max_size = max_size
self.net_timeout = net_timeout
self.conn_timeout = conn_timeout
self.wait_queue_timeout = wait_queue_timeout
self.wait_queue_multiple = wait_queue_multiple
self.use_ssl = use_ssl
self.ssl_keyfile = ssl_keyfile
self.ssl_certfile = ssl_certfile
self.ssl_cert_reqs = ssl_cert_reqs
self.ssl_ca_certs = ssl_ca_certs
if HAS_SSL and use_ssl and not ssl_cert_reqs:
self.ssl_cert_reqs = ssl.CERT_NONE
# Map self._ident.get() -> request socket
self._tid_to_sock = {}
if use_greenlets and not thread_util.have_gevent:
raise ConfigurationError("The Gevent module is not available. "
"Install the gevent package from PyPI.")
self._ident = thread_util.create_ident(use_greenlets)
# Count the number of calls to start_request() per thread or greenlet
self._request_counter = thread_util.Counter(use_greenlets)
if self.wait_queue_multiple is None or self.max_size is None:
max_waiters = None
else:
max_waiters = self.max_size * self.wait_queue_multiple
self._socket_semaphore = thread_util.create_semaphore(
self.max_size, max_waiters, use_greenlets)
def __init__(self,
pair,
max_size,
net_timeout,
conn_timeout,
use_ssl,
use_greenlets,
ssl_keyfile=None,
ssl_certfile=None,
ssl_cert_reqs=None,
ssl_ca_certs=None,
wait_queue_timeout=None,
wait_queue_multiple=None,
socket_keepalive=False,
ssl_match_hostname=True):
"""
:Parameters:
- `pair`: a (hostname, port) tuple
- `max_size`: The maximum number of open sockets. Calls to
`get_socket` will block if this is set, this pool has opened
`max_size` sockets, and there are none idle. Set to `None` to
disable.
- `net_timeout`: timeout in seconds for operations on open connection
- `conn_timeout`: timeout in seconds for establishing connection
- `use_ssl`: bool, if True use an encrypted connection
- `use_greenlets`: bool, if True then start_request() assigns a
socket to the current greenlet - otherwise it is assigned to the
current thread
- `ssl_keyfile`: The private keyfile used to identify the local
connection against mongod. If included with the ``certfile` then
only the ``ssl_certfile`` is needed. Implies ``ssl=True``.
- `ssl_certfile`: The certificate file used to identify the local
connection against mongod. Implies ``ssl=True``.
- `ssl_cert_reqs`: Specifies whether a certificate is required from
the other side of the connection, and whether it will be validated
if provided. It must be one of the three values ``ssl.CERT_NONE``
(certificates ignored), ``ssl.CERT_OPTIONAL``
(not required, but validated if provided), or ``ssl.CERT_REQUIRED``
(required and validated). If the value of this parameter is not
``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
to a file of CA certificates. Implies ``ssl=True``.
- `ssl_ca_certs`: The ca_certs file contains a set of concatenated
"certification authority" certificates, which are used to validate
certificates passed from the other end of the connection.
Implies ``ssl=True``.
- `wait_queue_timeout`: (integer) How long (in seconds) a
thread will wait for a socket from the pool if the pool has no
free sockets.
- `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
the number of threads allowed to wait for a socket at one time.
- `socket_keepalive`: (boolean) Whether to send periodic keep-alive
packets on connected sockets. Defaults to ``False`` (do not send
keep-alive packets).
- `ssl_match_hostname`: If ``True`` (the default), and
`ssl_cert_reqs` is not ``ssl.CERT_NONE``, enables hostname
verification using the :func:`~ssl.match_hostname` function from
python's :mod:`~ssl` module. Think very carefully before setting
this to ``False`` as that could make your application vulnerable to
man-in-the-middle attacks.
"""
# Only check a socket's health with _closed() every once in a while.
# Can override for testing: 0 to always check, None to never check.
self._check_interval_seconds = 1
self.sockets = set()
self.lock = threading.Lock()
# Keep track of resets, so we notice sockets created before the most
# recent reset and close them.
self.pool_id = 0
self.pid = os.getpid()
self.pair = pair
self.max_size = max_size
self.net_timeout = net_timeout
self.conn_timeout = conn_timeout
self.wait_queue_timeout = wait_queue_timeout
self.wait_queue_multiple = wait_queue_multiple
self.socket_keepalive = socket_keepalive
self.ssl_ctx = None
self.ssl_match_hostname = ssl_match_hostname
if HAS_SSL and use_ssl:
# PROTOCOL_TLS_CLIENT added and PROTOCOL_SSLv23
# deprecated in CPython 3.6
self.ssl_ctx = _SSLContext(
getattr(ssl, 'PROTOCOL_TLS_CLIENT', ssl.PROTOCOL_SSLv23))
# SSLContext.check_hostname was added in 2.7.9 and 3.4. Using it
# forces the use of SNI, which PyMongo 2.x doesn't support.
# PROTOCOL_TLS_CLIENT enables this by default. Since we call
# match_hostname directly disable this explicitly.
if hasattr(self.ssl_ctx, "check_hostname"):
self.ssl_ctx.check_hostname = False
# load_cert_chain and load_verify_locations can fail
# if the file contents are invalid.
if ssl_certfile is not None:
try:
self.ssl_ctx.load_cert_chain(ssl_certfile, ssl_keyfile)
except ssl.SSLError:
pass
if ssl_ca_certs is not None:
try:
self.ssl_ctx.load_verify_locations(ssl_ca_certs)
except ssl.SSLError:
pass
# PROTOCOL_TLS_CLIENT sets verify_mode to CERT_REQUIRED so
# we always have to set this explicitly.
if ssl_cert_reqs is not None:
self.ssl_ctx.verify_mode = ssl_cert_reqs
else:
self.ssl_ctx.verify_mode = ssl.CERT_NONE
# Map self._ident.get() -> request socket
self._tid_to_sock = {}
if use_greenlets and not thread_util.have_gevent:
raise ConfigurationError("The Gevent module is not available. "
"Install the gevent package from PyPI.")
self._ident = thread_util.create_ident(use_greenlets)
# Count the number of calls to start_request() per thread or greenlet
self._request_counter = thread_util.Counter(use_greenlets)
if self.wait_queue_multiple is None or self.max_size is None:
max_waiters = None
else:
max_waiters = self.max_size * self.wait_queue_multiple
self._socket_semaphore = thread_util.create_semaphore(
self.max_size, max_waiters, use_greenlets)
def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl,
use_greenlets, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs=None, ssl_ca_certs=None,
wait_queue_timeout=None, wait_queue_multiple=None):
"""
:Parameters:
- `pair`: a (hostname, port) tuple
- `max_size`: The maximum number of open sockets. Calls to
`get_socket` will block if this is set, this pool has opened
`max_size` sockets, and there are none idle. Set to `None` to
disable.
- `net_timeout`: timeout in seconds for operations on open connection
- `conn_timeout`: timeout in seconds for establishing connection
- `use_ssl`: bool, if True use an encrypted connection
- `use_greenlets`: bool, if True then start_request() assigns a
socket to the current greenlet - otherwise it is assigned to the
current thread
- `ssl_keyfile`: The private keyfile used to identify the local
connection against mongod. If included with the ``certfile` then
only the ``ssl_certfile`` is needed. Implies ``ssl=True``.
- `ssl_certfile`: The certificate file used to identify the local
connection against mongod. Implies ``ssl=True``.
- `ssl_cert_reqs`: Specifies whether a certificate is required from
the other side of the connection, and whether it will be validated
if provided. It must be one of the three values ``ssl.CERT_NONE``
(certificates ignored), ``ssl.CERT_OPTIONAL``
(not required, but validated if provided), or ``ssl.CERT_REQUIRED``
(required and validated). If the value of this parameter is not
``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
to a file of CA certificates. Implies ``ssl=True``.
- `ssl_ca_certs`: The ca_certs file contains a set of concatenated
"certification authority" certificates, which are used to validate
certificates passed from the other end of the connection.
Implies ``ssl=True``.
- `wait_queue_timeout`: (integer) How long (in seconds) a
thread will wait for a socket from the pool if the pool has no
free sockets.
- `wait_queue_multiple`: (integer) Multiplied by max_pool_size to give
the number of threads allowed to wait for a socket at one time.
"""
# Only check a socket's health with _closed() every once in a while.
# Can override for testing: 0 to always check, None to never check.
self._check_interval_seconds = 1
self.sockets = set()
self.lock = threading.Lock()
# Keep track of resets, so we notice sockets created before the most
# recent reset and close them.
self.pool_id = 0
self.pid = os.getpid()
self.pair = pair
self.max_size = max_size
self.net_timeout = net_timeout
self.conn_timeout = conn_timeout
self.wait_queue_timeout = wait_queue_timeout
self.wait_queue_multiple = wait_queue_multiple
self.use_ssl = use_ssl
self.ssl_keyfile = ssl_keyfile
self.ssl_certfile = ssl_certfile
self.ssl_cert_reqs = ssl_cert_reqs
self.ssl_ca_certs = ssl_ca_certs
if HAS_SSL and use_ssl and not ssl_cert_reqs:
self.ssl_cert_reqs = ssl.CERT_NONE
# Map self._ident.get() -> request socket
self._tid_to_sock = {}
if use_greenlets and not thread_util.have_gevent:
raise ConfigurationError(
"The Gevent module is not available. "
"Install the gevent package from PyPI."
)
self._ident = thread_util.create_ident(use_greenlets)
# Count the number of calls to start_request() per thread or greenlet
self._request_counter = thread_util.Counter(use_greenlets)
if self.wait_queue_multiple is None or self.max_size is None:
max_waiters = None
else:
max_waiters = self.max_size * self.wait_queue_multiple
self._socket_semaphore = thread_util.create_semaphore(
self.max_size, max_waiters, use_greenlets)
def __init__(self, pair, max_size, net_timeout, conn_timeout, use_ssl,
use_greenlets, ssl_keyfile=None, ssl_certfile=None,
ssl_cert_reqs=None, ssl_ca_certs=None):
"""
:Parameters:
- `pair`: a (hostname, port) tuple
- `max_size`: approximate number of idle connections to keep open
- `net_timeout`: timeout in seconds for operations on open connection
- `conn_timeout`: timeout in seconds for establishing connection
- `use_ssl`: bool, if True use an encrypted connection
- `use_greenlets`: bool, if True then start_request() assigns a
socket to the current greenlet - otherwise it is assigned to the
current thread
- `ssl_keyfile`: The private keyfile used to identify the local
connection against mongod. If included with the ``certfile` then
only the ``ssl_certfile`` is needed. Implies ``ssl=True``.
- `ssl_certfile`: The certificate file used to identify the local
connection against mongod. Implies ``ssl=True``.
- `ssl_cert_reqs`: Specifies whether a certificate is required from
the other side of the connection, and whether it will be validated
if provided. It must be one of the three values ``ssl.CERT_NONE``
(certificates ignored), ``ssl.CERT_OPTIONAL``
(not required, but validated if provided), or ``ssl.CERT_REQUIRED``
(required and validated). If the value of this parameter is not
``ssl.CERT_NONE``, then the ``ssl_ca_certs`` parameter must point
to a file of CA certificates. Implies ``ssl=True``.
- `ssl_ca_certs`: The ca_certs file contains a set of concatenated
"certification authority" certificates, which are used to validate
certificates passed from the other end of the connection.
Implies ``ssl=True``.
"""
if use_greenlets and not thread_util.have_greenlet:
raise ConfigurationError(
"The greenlet module is not available. "
"Install the greenlet package from PyPI."
)
self.sockets = set()
self.lock = threading.Lock()
# Keep track of resets, so we notice sockets created before the most
# recent reset and close them.
self.pool_id = 0
self.pid = os.getpid()
self.pair = pair
self.max_size = max_size
self.net_timeout = net_timeout
self.conn_timeout = conn_timeout
self.use_ssl = use_ssl
self.ssl_keyfile = ssl_keyfile
self.ssl_certfile = ssl_certfile
self.ssl_cert_reqs = ssl_cert_reqs
self.ssl_ca_certs = ssl_ca_certs
if HAS_SSL and use_ssl and not ssl_cert_reqs:
self.ssl_cert_reqs = ssl.CERT_NONE
self._ident = thread_util.create_ident(use_greenlets)
# Map self._ident.get() -> request socket
self._tid_to_sock = {}
# Count the number of calls to start_request() per thread or greenlet
self._request_counter = thread_util.Counter(use_greenlets)
def _test_ident(self, use_greenlets):
ident = thread_util.create_ident(use_greenlets)
ids = set([ident.get()])
unwatched_id = []
done = set([ident.get()]) # Start with main thread's / greenlet's id.
died = set()
class Watched(object):
def __init__(self, ident):
self._my_ident = ident
def before_rendezvous(self):
self.my_id = self._my_ident.get()
ids.add(self.my_id)
def after_rendezvous(self):
assert not self._my_ident.watching()
self._my_ident.watch(lambda ref: died.add(self.my_id))
assert self._my_ident.watching()
done.add(self.my_id)
class Unwatched(Watched):
def before_rendezvous(self):
Watched.before_rendezvous(self)
unwatched_id.append(self.my_id)
def after_rendezvous(self):
Watched.after_rendezvous(self)
self._my_ident.unwatch(self.my_id)
assert not self._my_ident.watching()
if use_greenlets:
class WatchedGreenlet(Watched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
class UnwatchedGreenlet(Unwatched):
def run(self):
self.before_rendezvous()
self.after_rendezvous()
t_watched = greenlet.greenlet(WatchedGreenlet(ident).run)
t_unwatched = greenlet.greenlet(UnwatchedGreenlet(ident).run)
looplet([t_watched, t_unwatched])
else:
class WatchedThread(Watched, RendezvousThread):
def __init__(self, ident, state):
Watched.__init__(self, ident)
RendezvousThread.__init__(self, state)
class UnwatchedThread(Unwatched, RendezvousThread):
def __init__(self, ident, state):
Unwatched.__init__(self, ident)
RendezvousThread.__init__(self, state)
state = RendezvousThread.create_shared_state(2)
t_watched = WatchedThread(ident, state)
t_watched.start()
t_unwatched = UnwatchedThread(ident, state)
t_unwatched.start()
RendezvousThread.wait_for_rendezvous(state)
RendezvousThread.resume_after_rendezvous(state)
t_watched.join()
t_unwatched.join()
self.assertTrue(t_watched.passed)
self.assertTrue(t_unwatched.passed)
# Remove references, let weakref callbacks run
del t_watched
del t_unwatched
# Trigger final cleanup in Python <= 2.7.0.
# http://bugs.python.org/issue1868
ident.get()
self.assertEqual(3, len(ids))
self.assertEqual(3, len(done))
# Make sure thread is really gone
slept = 0
while not died and slept < 10:
time.sleep(1)
gc.collect()
slept += 1
self.assertEqual(1, len(died))
self.assertFalse(unwatched_id[0] in died)